{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use Soft Actor-Critic with Auto $\\alpha$ Tuning to Play LunarLanderContinuous-v2\n",
    "\n",
    "PyTorch version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "import copy\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "torch.manual_seed(0)\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.distributions as distributions\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:18:11 [INFO] env: <LunarLanderContinuous<LunarLanderContinuous-v2>>\n",
      "08:18:11 [INFO] action_space: Box(-1.0, 1.0, (2,), float32)\n",
      "08:18:11 [INFO] observation_space: Box(-inf, inf, (8,), float32)\n",
      "08:18:11 [INFO] reward_range: (-inf, inf)\n",
      "08:18:11 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 50}\n",
      "08:18:11 [INFO] _max_episode_steps: 1000\n",
      "08:18:11 [INFO] _elapsed_steps: None\n",
      "08:18:11 [INFO] id: LunarLanderContinuous-v2\n",
      "08:18:11 [INFO] entry_point: gym.envs.box2d:LunarLanderContinuous\n",
      "08:18:11 [INFO] reward_threshold: 200\n",
      "08:18:11 [INFO] nondeterministic: False\n",
      "08:18:11 [INFO] max_episode_steps: 1000\n",
      "08:18:11 [INFO] _kwargs: {}\n",
      "08:18:11 [INFO] _env_name: LunarLanderContinuous\n"
     ]
    }
   ],
   "source": [
    "env = gym.make(\"LunarLanderContinuous-v2\")\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SACAgent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_dim = env.action_space.shape[0]\n",
    "        self.action_low = env.action_space.low\n",
    "        self.action_high = env.action_space.high\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(100000)\n",
    "\n",
    "        # create alpha\n",
    "        self.target_entropy = -self.action_dim\n",
    "        self.ln_alpha_tensor = torch.zeros(1, requires_grad=True)\n",
    "        self.alpha_optimizer = optim.Adam([self.ln_alpha_tensor,], lr=0.0003)\n",
    "\n",
    "        # create actor\n",
    "        self.actor_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256], output_size=self.action_dim*2,\n",
    "                output_activator=nn.Tanh())\n",
    "        self.actor_optimizier = optim.Adam(self.actor_net.parameters(), lr=0.0003)\n",
    "\n",
    "        # create V critic\n",
    "        self.v_evaluate_net = self.build_net(input_size=state_dim,\n",
    "                hidden_sizes=[256, 256])\n",
    "        self.v_target_net = copy.deepcopy(self.v_evaluate_net)\n",
    "        self.v_loss = nn.MSELoss()\n",
    "        self.v_optimizer = optim.Adam(self.v_evaluate_net.parameters(), lr=0.0003)\n",
    "\n",
    "        # create Q critic\n",
    "        self.q0_net = self.build_net(input_size=state_dim+self.action_dim,\n",
    "                hidden_sizes=[256, 256])\n",
    "        self.q1_net = self.build_net(input_size=state_dim+self.action_dim,\n",
    "                hidden_sizes=[256, 256])\n",
    "        self.q0_loss = nn.MSELoss()\n",
    "        self.q1_loss = nn.MSELoss()\n",
    "        self.q0_optimizer = optim.Adam(self.q0_net.parameters(), lr=0.0003)\n",
    "        self.q1_optimizer = optim.Adam(self.q1_net.parameters(), lr=0.0003)\n",
    "\n",
    "    def build_net(self, input_size, hidden_sizes, output_size=1,\n",
    "            output_activator=None):\n",
    "        layers = []\n",
    "        for input_size, output_size in zip(\n",
    "                [input_size,] + hidden_sizes, hidden_sizes + [output_size,]):\n",
    "            layers.append(nn.Linear(input_size, output_size))\n",
    "            layers.append(nn.ReLU())\n",
    "        layers = layers[:-1]\n",
    "        if output_activator:\n",
    "            layers.append(output_activator)\n",
    "        net = nn.Sequential(*layers)\n",
    "        return net\n",
    "\n",
    "    def get_action_ln_prob_tensors(self, state_tensor):\n",
    "        mean_ln_std_tensor = self.actor_net(state_tensor)\n",
    "        mean_tensor, ln_std_tensor = torch.split(mean_ln_std_tensor,\n",
    "                self.action_dim, dim=-1)\n",
    "        if self.mode == 'train':\n",
    "            std_tensor = torch.exp(ln_std_tensor)\n",
    "            normal_dist = distributions.Normal(mean_tensor, std_tensor)\n",
    "            rsample_tensor = normal_dist.rsample()\n",
    "            action_tensor = torch.tanh(rsample_tensor)\n",
    "            ln_prob_tensor = normal_dist.log_prob(rsample_tensor) - \\\n",
    "                    torch.log1p(1e-6 - action_tensor.pow(2))\n",
    "            ln_prob_tensor = ln_prob_tensor.sum(-1, keepdim=True)\n",
    "        else:\n",
    "            action_tensor = torch.tanh(mean_tensor)\n",
    "            ln_prob_tensor = torch.ones_like(action_tensor)\n",
    "        return action_tensor, ln_prob_tensor\n",
    "\n",
    "    def reset(self, mode):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and self.replayer.count < 5000:\n",
    "            action = np.random.uniform(self.action_low, self.action_high)\n",
    "        else:\n",
    "            state_tensor = torch.as_tensor(observation, dtype=torch.float\n",
    "                    ).unsqueeze(0)\n",
    "            action_tensor, _ = self.get_action_ln_prob_tensors(state_tensor)\n",
    "            action = action_tensor[0].detach().numpy()\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = \\\n",
    "                        self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "            if self.replayer.count >= 128:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.005):\n",
    "        for target_param, evaluate_param in zip(\n",
    "                target_net.parameters(), evaluate_net.parameters()):\n",
    "            target_param.data.copy_(learning_rate * evaluate_param.data\n",
    "                    + (1 - learning_rate) * target_param.data)\n",
    "\n",
    "    def learn(self):\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(128)\n",
    "        state_tensor = torch.as_tensor(states, dtype=torch.float)\n",
    "        action_tensor = torch.as_tensor(actions, dtype=torch.float)\n",
    "        reward_tensor = torch.as_tensor(rewards, dtype=torch.float)\n",
    "        next_state_tensor = torch.as_tensor(next_states, dtype=torch.float)\n",
    "        done_tensor = torch.as_tensor(dones, dtype=torch.float)\n",
    "\n",
    "        # train alpha\n",
    "        act_tensor, ln_prob_tensor = self.get_action_ln_prob_tensors(state_tensor)\n",
    "        alpha_loss_tensor = (-self.ln_alpha_tensor * (ln_prob_tensor +\n",
    "                self.target_entropy).detach()).mean()\n",
    "\n",
    "        self.alpha_optimizer.zero_grad()\n",
    "        alpha_loss_tensor.backward()\n",
    "        self.alpha_optimizer.step()\n",
    "\n",
    "        # train Q critic\n",
    "        states_action_tensor = torch.cat((state_tensor, action_tensor), dim=-1)\n",
    "        q0_tensor = self.q0_net(states_action_tensor)\n",
    "        q1_tensor = self.q1_net(states_action_tensor)\n",
    "        next_v_tensor = self.v_target_net(next_state_tensor)\n",
    "        q_target = reward_tensor.unsqueeze(1) + \\\n",
    "                self.gamma * next_v_tensor * (1. - done_tensor.unsqueeze(1))\n",
    "        q0_loss_tensor = self.q0_loss(q0_tensor, q_target.detach())\n",
    "        q1_loss_tensor = self.q1_loss(q1_tensor, q_target.detach())\n",
    "\n",
    "        self.q0_optimizer.zero_grad()\n",
    "        q0_loss_tensor.backward()\n",
    "        self.q0_optimizer.step()\n",
    "\n",
    "        self.q1_optimizer.zero_grad()\n",
    "        q1_loss_tensor.backward()\n",
    "        self.q1_optimizer.step()\n",
    "\n",
    "        # train V critic\n",
    "        state_act_tensor = torch.cat((state_tensor, act_tensor), dim=-1)\n",
    "        v_pred_tensor = self.v_evaluate_net(state_tensor)\n",
    "        q0_pred_tensor = self.q0_net(state_act_tensor)\n",
    "        q1_pred_tensor = self.q1_net(state_act_tensor)\n",
    "        q_pred_tensor = torch.min(q0_pred_tensor, q1_pred_tensor)\n",
    "        alpha_tensor = self.ln_alpha_tensor.exp()\n",
    "        v_target_tensor = q_pred_tensor - alpha_tensor * ln_prob_tensor\n",
    "        v_loss_tensor = self.v_loss(v_pred_tensor, v_target_tensor.detach())\n",
    "\n",
    "        self.v_optimizer.zero_grad()\n",
    "        v_loss_tensor.backward()\n",
    "        self.v_optimizer.step()\n",
    "\n",
    "        self.update_net(self.v_target_net, self.v_evaluate_net)\n",
    "\n",
    "        # train actor\n",
    "        actor_loss_tensor = (alpha_tensor * ln_prob_tensor\n",
    "                - q0_pred_tensor).mean()\n",
    "\n",
    "        self.actor_optimizier.zero_grad()\n",
    "        actor_loss_tensor.backward()\n",
    "        self.actor_optimizier.step()\n",
    "\n",
    "\n",
    "agent = SACAgent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "08:18:11 [INFO] ==== train ====\n",
      "08:18:12 [DEBUG] train episode 0: reward = -112.08, steps = 140\n",
      "08:18:18 [DEBUG] train episode 1: reward = -281.50, steps = 159\n",
      "08:18:23 [DEBUG] train episode 2: reward = -234.58, steps = 137\n",
      "08:18:30 [DEBUG] train episode 3: reward = -158.37, steps = 187\n",
      "08:18:37 [DEBUG] train episode 4: reward = -298.07, steps = 170\n",
      "08:18:40 [DEBUG] train episode 5: reward = -335.86, steps = 75\n",
      "08:18:44 [DEBUG] train episode 6: reward = -426.55, steps = 108\n",
      "08:18:48 [DEBUG] train episode 7: reward = -149.63, steps = 111\n",
      "08:18:51 [DEBUG] train episode 8: reward = -69.06, steps = 73\n",
      "08:18:55 [DEBUG] train episode 9: reward = -271.07, steps = 96\n",
      "08:18:59 [DEBUG] train episode 10: reward = -408.78, steps = 105\n",
      "08:19:04 [DEBUG] train episode 11: reward = -102.08, steps = 137\n",
      "08:19:10 [DEBUG] train episode 12: reward = -390.09, steps = 145\n",
      "08:19:15 [DEBUG] train episode 13: reward = -408.79, steps = 107\n",
      "08:19:17 [DEBUG] train episode 14: reward = -92.82, steps = 69\n",
      "08:19:21 [DEBUG] train episode 15: reward = -352.78, steps = 87\n",
      "08:19:24 [DEBUG] train episode 16: reward = -234.04, steps = 89\n",
      "08:19:29 [DEBUG] train episode 17: reward = -250.68, steps = 127\n",
      "08:19:34 [DEBUG] train episode 18: reward = -250.59, steps = 119\n",
      "08:19:40 [DEBUG] train episode 19: reward = -320.67, steps = 152\n",
      "08:19:43 [DEBUG] train episode 20: reward = -373.46, steps = 81\n",
      "08:19:46 [DEBUG] train episode 21: reward = -88.18, steps = 83\n",
      "08:19:49 [DEBUG] train episode 22: reward = -36.75, steps = 80\n",
      "08:19:53 [DEBUG] train episode 23: reward = -139.52, steps = 97\n",
      "08:19:57 [DEBUG] train episode 24: reward = -352.28, steps = 106\n",
      "08:20:02 [DEBUG] train episode 25: reward = -111.02, steps = 121\n",
      "08:20:06 [DEBUG] train episode 26: reward = -173.87, steps = 104\n",
      "08:20:10 [DEBUG] train episode 27: reward = -357.54, steps = 101\n",
      "08:20:15 [DEBUG] train episode 28: reward = -187.84, steps = 125\n",
      "08:20:23 [DEBUG] train episode 29: reward = -259.05, steps = 213\n",
      "08:20:27 [DEBUG] train episode 30: reward = -189.09, steps = 113\n",
      "08:20:31 [DEBUG] train episode 31: reward = -233.59, steps = 106\n",
      "08:20:35 [DEBUG] train episode 32: reward = -447.59, steps = 104\n",
      "08:20:41 [DEBUG] train episode 33: reward = -298.22, steps = 157\n",
      "08:20:46 [DEBUG] train episode 34: reward = -312.16, steps = 106\n",
      "08:20:50 [DEBUG] train episode 35: reward = -99.37, steps = 114\n",
      "08:20:54 [DEBUG] train episode 36: reward = -402.26, steps = 115\n",
      "08:20:58 [DEBUG] train episode 37: reward = -280.93, steps = 104\n",
      "08:21:02 [DEBUG] train episode 38: reward = -77.80, steps = 104\n",
      "08:21:05 [DEBUG] train episode 39: reward = -194.40, steps = 82\n",
      "08:21:10 [DEBUG] train episode 40: reward = -260.22, steps = 120\n",
      "08:21:15 [DEBUG] train episode 41: reward = -267.42, steps = 117\n",
      "08:21:20 [DEBUG] train episode 42: reward = -116.98, steps = 134\n",
      "08:21:30 [DEBUG] train episode 43: reward = -338.80, steps = 225\n",
      "08:21:40 [DEBUG] train episode 44: reward = -270.31, steps = 233\n",
      "08:21:47 [DEBUG] train episode 45: reward = -133.80, steps = 155\n",
      "08:21:54 [DEBUG] train episode 46: reward = -63.22, steps = 150\n",
      "08:22:01 [DEBUG] train episode 47: reward = -290.41, steps = 176\n",
      "08:22:12 [DEBUG] train episode 48: reward = -46.56, steps = 252\n",
      "08:22:30 [DEBUG] train episode 49: reward = -115.06, steps = 431\n",
      "08:22:46 [DEBUG] train episode 50: reward = -102.78, steps = 392\n",
      "08:23:29 [DEBUG] train episode 51: reward = -150.02, steps = 1000\n",
      "08:24:12 [DEBUG] train episode 52: reward = -58.18, steps = 1000\n",
      "08:24:55 [DEBUG] train episode 53: reward = -137.97, steps = 1000\n",
      "08:25:16 [DEBUG] train episode 54: reward = 189.33, steps = 514\n",
      "08:25:58 [DEBUG] train episode 55: reward = -79.11, steps = 1000\n",
      "08:26:41 [DEBUG] train episode 56: reward = -62.32, steps = 1000\n",
      "08:27:09 [DEBUG] train episode 57: reward = -146.16, steps = 657\n",
      "08:27:18 [DEBUG] train episode 58: reward = -310.16, steps = 233\n",
      "08:27:26 [DEBUG] train episode 59: reward = -141.95, steps = 207\n",
      "08:27:33 [DEBUG] train episode 60: reward = -24.10, steps = 171\n",
      "08:27:39 [DEBUG] train episode 61: reward = 10.28, steps = 156\n",
      "08:27:47 [DEBUG] train episode 62: reward = -72.96, steps = 177\n",
      "08:28:09 [DEBUG] train episode 63: reward = 189.82, steps = 562\n",
      "08:28:19 [DEBUG] train episode 64: reward = -45.08, steps = 249\n",
      "08:28:36 [DEBUG] train episode 65: reward = 202.26, steps = 395\n",
      "08:29:18 [DEBUG] train episode 66: reward = -1.59, steps = 1000\n",
      "08:29:49 [DEBUG] train episode 67: reward = 85.66, steps = 765\n",
      "08:30:31 [DEBUG] train episode 68: reward = 49.17, steps = 1000\n",
      "08:31:14 [DEBUG] train episode 69: reward = -9.73, steps = 1000\n",
      "08:31:59 [DEBUG] train episode 70: reward = -43.56, steps = 1000\n",
      "08:32:32 [DEBUG] train episode 71: reward = 92.58, steps = 775\n",
      "08:33:17 [DEBUG] train episode 72: reward = 104.25, steps = 1000\n",
      "08:34:00 [DEBUG] train episode 73: reward = -2.90, steps = 1000\n",
      "08:34:44 [DEBUG] train episode 74: reward = -16.13, steps = 1000\n",
      "08:35:27 [DEBUG] train episode 75: reward = -81.60, steps = 1000\n",
      "08:36:12 [DEBUG] train episode 76: reward = -51.90, steps = 1000\n",
      "08:36:56 [DEBUG] train episode 77: reward = -34.58, steps = 1000\n",
      "08:37:39 [DEBUG] train episode 78: reward = -8.94, steps = 1000\n",
      "08:38:21 [DEBUG] train episode 79: reward = -138.96, steps = 992\n",
      "08:39:03 [DEBUG] train episode 80: reward = -37.00, steps = 1000\n",
      "08:39:45 [DEBUG] train episode 81: reward = -26.09, steps = 1000\n",
      "08:40:29 [DEBUG] train episode 82: reward = -38.16, steps = 1000\n",
      "08:41:11 [DEBUG] train episode 83: reward = -28.83, steps = 1000\n",
      "08:41:54 [DEBUG] train episode 84: reward = -32.24, steps = 1000\n",
      "08:42:31 [DEBUG] train episode 85: reward = 1.92, steps = 1000\n",
      "08:43:09 [DEBUG] train episode 86: reward = -41.07, steps = 1000\n",
      "08:43:49 [DEBUG] train episode 87: reward = -77.63, steps = 1000\n",
      "08:44:24 [DEBUG] train episode 88: reward = -144.59, steps = 926\n",
      "08:45:02 [DEBUG] train episode 89: reward = -51.94, steps = 1000\n",
      "08:45:40 [DEBUG] train episode 90: reward = -64.39, steps = 1000\n",
      "08:46:19 [DEBUG] train episode 91: reward = -30.99, steps = 1000\n",
      "08:46:56 [DEBUG] train episode 92: reward = -35.70, steps = 1000\n",
      "08:47:34 [DEBUG] train episode 93: reward = 40.63, steps = 1000\n",
      "08:48:11 [DEBUG] train episode 94: reward = -48.49, steps = 1000\n",
      "08:48:51 [DEBUG] train episode 95: reward = 14.42, steps = 1000\n",
      "08:49:30 [DEBUG] train episode 96: reward = -56.95, steps = 1000\n",
      "08:50:07 [DEBUG] train episode 97: reward = -1.87, steps = 1000\n",
      "08:50:44 [DEBUG] train episode 98: reward = -29.84, steps = 1000\n",
      "08:51:23 [DEBUG] train episode 99: reward = -29.44, steps = 1000\n",
      "08:52:00 [DEBUG] train episode 100: reward = -3.35, steps = 1000\n",
      "08:52:40 [DEBUG] train episode 101: reward = 4.47, steps = 1000\n",
      "08:53:17 [DEBUG] train episode 102: reward = -28.23, steps = 1000\n",
      "08:53:59 [DEBUG] train episode 103: reward = -36.16, steps = 1000\n",
      "08:54:18 [DEBUG] train episode 104: reward = -80.85, steps = 499\n",
      "08:54:55 [DEBUG] train episode 105: reward = -1.22, steps = 1000\n",
      "08:55:24 [DEBUG] train episode 106: reward = -147.22, steps = 761\n",
      "08:55:45 [DEBUG] train episode 107: reward = -113.34, steps = 574\n",
      "08:56:21 [DEBUG] train episode 108: reward = -236.57, steps = 919\n",
      "08:56:53 [DEBUG] train episode 109: reward = -208.71, steps = 845\n",
      "08:57:31 [DEBUG] train episode 110: reward = -76.35, steps = 1000\n",
      "08:58:10 [DEBUG] train episode 111: reward = -48.00, steps = 1000\n",
      "08:58:51 [DEBUG] train episode 112: reward = -35.55, steps = 1000\n",
      "08:59:32 [DEBUG] train episode 113: reward = -36.37, steps = 1000\n",
      "09:00:12 [DEBUG] train episode 114: reward = -24.10, steps = 1000\n",
      "09:00:53 [DEBUG] train episode 115: reward = -8.93, steps = 1000\n",
      "09:01:34 [DEBUG] train episode 116: reward = -32.26, steps = 1000\n",
      "09:02:16 [DEBUG] train episode 117: reward = -20.86, steps = 1000\n",
      "09:02:55 [DEBUG] train episode 118: reward = -23.36, steps = 1000\n",
      "09:03:35 [DEBUG] train episode 119: reward = -19.84, steps = 1000\n",
      "09:04:16 [DEBUG] train episode 120: reward = 4.26, steps = 1000\n",
      "09:04:57 [DEBUG] train episode 121: reward = 13.64, steps = 1000\n",
      "09:05:37 [DEBUG] train episode 122: reward = -49.42, steps = 1000\n",
      "09:06:20 [DEBUG] train episode 123: reward = 12.37, steps = 1000\n",
      "09:07:04 [DEBUG] train episode 124: reward = -3.07, steps = 1000\n",
      "09:07:39 [DEBUG] train episode 125: reward = 192.29, steps = 816\n",
      "09:08:20 [DEBUG] train episode 126: reward = -24.69, steps = 1000\n",
      "09:09:05 [DEBUG] train episode 127: reward = -73.62, steps = 1000\n",
      "09:09:46 [DEBUG] train episode 128: reward = 69.85, steps = 1000\n",
      "09:10:27 [DEBUG] train episode 129: reward = -47.97, steps = 1000\n",
      "09:11:09 [DEBUG] train episode 130: reward = -33.53, steps = 1000\n",
      "09:11:53 [DEBUG] train episode 131: reward = -11.22, steps = 1000\n",
      "09:12:24 [DEBUG] train episode 132: reward = -98.74, steps = 753\n",
      "09:13:07 [DEBUG] train episode 133: reward = -31.19, steps = 1000\n",
      "09:13:51 [DEBUG] train episode 134: reward = -25.83, steps = 1000\n",
      "09:14:34 [DEBUG] train episode 135: reward = -33.07, steps = 1000\n",
      "09:15:15 [DEBUG] train episode 136: reward = -47.70, steps = 1000\n",
      "09:15:59 [DEBUG] train episode 137: reward = 73.33, steps = 1000\n",
      "09:16:44 [DEBUG] train episode 138: reward = -27.60, steps = 1000\n",
      "09:17:29 [DEBUG] train episode 139: reward = -2.45, steps = 1000\n",
      "09:18:11 [DEBUG] train episode 140: reward = 30.67, steps = 1000\n",
      "09:18:53 [DEBUG] train episode 141: reward = 3.14, steps = 1000\n",
      "09:19:36 [DEBUG] train episode 142: reward = -11.03, steps = 1000\n",
      "09:20:21 [DEBUG] train episode 143: reward = -8.69, steps = 1000\n",
      "09:21:05 [DEBUG] train episode 144: reward = -23.17, steps = 1000\n",
      "09:21:53 [DEBUG] train episode 145: reward = -19.40, steps = 1000\n",
      "09:22:37 [DEBUG] train episode 146: reward = 63.90, steps = 1000\n",
      "09:23:20 [DEBUG] train episode 147: reward = -27.68, steps = 1000\n",
      "09:24:02 [DEBUG] train episode 148: reward = -14.27, steps = 1000\n",
      "09:24:45 [DEBUG] train episode 149: reward = -24.26, steps = 1000\n",
      "09:25:28 [DEBUG] train episode 150: reward = -26.83, steps = 1000\n",
      "09:26:13 [DEBUG] train episode 151: reward = -27.14, steps = 1000\n",
      "09:26:56 [DEBUG] train episode 152: reward = -15.51, steps = 1000\n",
      "09:27:39 [DEBUG] train episode 153: reward = -34.58, steps = 1000\n",
      "09:28:21 [DEBUG] train episode 154: reward = 37.20, steps = 1000\n",
      "09:29:03 [DEBUG] train episode 155: reward = 34.22, steps = 1000\n",
      "09:29:48 [DEBUG] train episode 156: reward = -5.36, steps = 1000\n",
      "09:30:31 [DEBUG] train episode 157: reward = -50.48, steps = 1000\n",
      "09:31:13 [DEBUG] train episode 158: reward = 77.25, steps = 1000\n",
      "09:31:28 [DEBUG] train episode 159: reward = -64.55, steps = 361\n",
      "09:32:12 [DEBUG] train episode 160: reward = -50.94, steps = 1000\n",
      "09:32:54 [DEBUG] train episode 161: reward = -24.62, steps = 1000\n",
      "09:33:38 [DEBUG] train episode 162: reward = -39.87, steps = 1000\n",
      "09:34:22 [DEBUG] train episode 163: reward = 113.43, steps = 1000\n",
      "09:35:05 [DEBUG] train episode 164: reward = -18.25, steps = 1000\n",
      "09:35:52 [DEBUG] train episode 165: reward = -32.68, steps = 1000\n",
      "09:36:19 [DEBUG] train episode 166: reward = 251.94, steps = 639\n",
      "09:37:03 [DEBUG] train episode 167: reward = 132.67, steps = 996\n",
      "09:37:48 [DEBUG] train episode 168: reward = 26.10, steps = 1000\n",
      "09:38:31 [DEBUG] train episode 169: reward = -23.84, steps = 1000\n",
      "09:39:16 [DEBUG] train episode 170: reward = -34.61, steps = 1000\n",
      "09:39:59 [DEBUG] train episode 171: reward = 16.13, steps = 1000\n",
      "09:40:10 [DEBUG] train episode 172: reward = -28.65, steps = 267\n",
      "09:40:55 [DEBUG] train episode 173: reward = -6.24, steps = 1000\n",
      "09:41:41 [DEBUG] train episode 174: reward = -37.24, steps = 1000\n",
      "09:42:17 [DEBUG] train episode 175: reward = 153.42, steps = 838\n",
      "09:42:48 [DEBUG] train episode 176: reward = -106.54, steps = 704\n",
      "09:43:32 [DEBUG] train episode 177: reward = -15.26, steps = 1000\n",
      "09:44:15 [DEBUG] train episode 178: reward = -33.40, steps = 1000\n",
      "09:45:01 [DEBUG] train episode 179: reward = -33.85, steps = 1000\n",
      "09:45:47 [DEBUG] train episode 180: reward = -15.62, steps = 1000\n",
      "09:46:34 [DEBUG] train episode 181: reward = -26.81, steps = 1000\n",
      "09:47:19 [DEBUG] train episode 182: reward = 0.83, steps = 1000\n",
      "09:48:04 [DEBUG] train episode 183: reward = 128.82, steps = 981\n",
      "09:48:49 [DEBUG] train episode 184: reward = -46.66, steps = 1000\n",
      "09:49:35 [DEBUG] train episode 185: reward = -45.17, steps = 1000\n",
      "09:50:22 [DEBUG] train episode 186: reward = -39.43, steps = 1000\n",
      "09:51:06 [DEBUG] train episode 187: reward = 79.79, steps = 1000\n",
      "09:51:53 [DEBUG] train episode 188: reward = -23.49, steps = 1000\n",
      "09:52:38 [DEBUG] train episode 189: reward = -37.61, steps = 1000\n",
      "09:53:22 [DEBUG] train episode 190: reward = -12.40, steps = 1000\n",
      "09:54:06 [DEBUG] train episode 191: reward = -10.87, steps = 1000\n",
      "09:54:48 [DEBUG] train episode 192: reward = -18.93, steps = 1000\n",
      "09:55:35 [DEBUG] train episode 193: reward = -30.56, steps = 1000\n",
      "09:56:21 [DEBUG] train episode 194: reward = -40.22, steps = 1000\n",
      "09:57:09 [DEBUG] train episode 195: reward = -10.30, steps = 1000\n",
      "09:57:55 [DEBUG] train episode 196: reward = -1.69, steps = 1000\n",
      "09:58:42 [DEBUG] train episode 197: reward = 10.05, steps = 1000\n",
      "09:59:23 [DEBUG] train episode 198: reward = 167.86, steps = 914\n",
      "10:00:10 [DEBUG] train episode 199: reward = 15.12, steps = 1000\n",
      "10:00:54 [DEBUG] train episode 200: reward = -11.66, steps = 1000\n",
      "10:01:40 [DEBUG] train episode 201: reward = -19.04, steps = 1000\n",
      "10:02:06 [DEBUG] train episode 202: reward = 245.18, steps = 589\n",
      "10:02:54 [DEBUG] train episode 203: reward = 32.02, steps = 1000\n",
      "10:03:42 [DEBUG] train episode 204: reward = 56.35, steps = 1000\n",
      "10:04:20 [DEBUG] train episode 205: reward = 186.69, steps = 836\n",
      "10:05:08 [DEBUG] train episode 206: reward = -14.67, steps = 1000\n",
      "10:05:41 [DEBUG] train episode 207: reward = 203.12, steps = 736\n",
      "10:05:57 [DEBUG] train episode 208: reward = 262.30, steps = 366\n",
      "10:06:29 [DEBUG] train episode 209: reward = 265.60, steps = 726\n",
      "10:07:11 [DEBUG] train episode 210: reward = 180.46, steps = 914\n",
      "10:07:36 [DEBUG] train episode 211: reward = 188.73, steps = 580\n",
      "10:08:05 [DEBUG] train episode 212: reward = 170.45, steps = 661\n",
      "10:08:24 [DEBUG] train episode 213: reward = 278.07, steps = 428\n",
      "10:08:48 [DEBUG] train episode 214: reward = 276.87, steps = 559\n",
      "10:09:33 [DEBUG] train episode 215: reward = -13.98, steps = 1000\n",
      "10:09:57 [DEBUG] train episode 216: reward = 215.50, steps = 537\n",
      "10:10:43 [DEBUG] train episode 217: reward = 1.84, steps = 1000\n",
      "10:11:11 [DEBUG] train episode 218: reward = 217.29, steps = 636\n",
      "10:11:34 [DEBUG] train episode 219: reward = 255.23, steps = 514\n",
      "10:12:01 [DEBUG] train episode 220: reward = 186.56, steps = 597\n",
      "10:12:48 [DEBUG] train episode 221: reward = 81.03, steps = 1000\n",
      "10:13:25 [DEBUG] train episode 222: reward = 153.45, steps = 857\n",
      "10:13:31 [DEBUG] train episode 223: reward = 18.37, steps = 123\n",
      "10:14:03 [DEBUG] train episode 224: reward = 275.56, steps = 744\n",
      "10:14:34 [DEBUG] train episode 225: reward = 240.97, steps = 700\n",
      "10:15:01 [DEBUG] train episode 226: reward = 274.98, steps = 623\n",
      "10:15:07 [DEBUG] train episode 227: reward = 20.80, steps = 144\n",
      "10:15:14 [DEBUG] train episode 228: reward = -1.63, steps = 166\n",
      "10:15:31 [DEBUG] train episode 229: reward = 233.82, steps = 387\n",
      "10:15:47 [DEBUG] train episode 230: reward = 259.04, steps = 363\n",
      "10:16:33 [DEBUG] train episode 231: reward = 107.03, steps = 1000\n",
      "10:16:54 [DEBUG] train episode 232: reward = 224.22, steps = 488\n",
      "10:17:20 [DEBUG] train episode 233: reward = 205.25, steps = 563\n",
      "10:18:07 [DEBUG] train episode 234: reward = 4.50, steps = 1000\n",
      "10:18:55 [DEBUG] train episode 235: reward = 119.17, steps = 993\n",
      "10:19:41 [DEBUG] train episode 236: reward = -31.98, steps = 1000\n",
      "10:20:28 [DEBUG] train episode 237: reward = -49.84, steps = 1000\n",
      "10:21:13 [DEBUG] train episode 238: reward = -43.70, steps = 1000\n",
      "10:21:54 [DEBUG] train episode 239: reward = -112.50, steps = 860\n",
      "10:22:38 [DEBUG] train episode 240: reward = 158.07, steps = 1000\n",
      "10:23:22 [DEBUG] train episode 241: reward = -66.71, steps = 1000\n",
      "10:23:44 [DEBUG] train episode 242: reward = 222.56, steps = 490\n",
      "10:24:32 [DEBUG] train episode 243: reward = -27.62, steps = 1000\n",
      "10:24:44 [DEBUG] train episode 244: reward = 273.56, steps = 298\n",
      "10:25:32 [DEBUG] train episode 245: reward = -53.45, steps = 1000\n",
      "10:26:17 [DEBUG] train episode 246: reward = -3.71, steps = 1000\n",
      "10:26:42 [DEBUG] train episode 247: reward = 204.70, steps = 562\n",
      "10:27:12 [DEBUG] train episode 248: reward = 210.68, steps = 668\n",
      "10:27:57 [DEBUG] train episode 249: reward = -24.15, steps = 1000\n",
      "10:28:13 [DEBUG] train episode 250: reward = 236.60, steps = 371\n",
      "10:28:26 [DEBUG] train episode 251: reward = 275.84, steps = 307\n",
      "10:29:00 [DEBUG] train episode 252: reward = 214.70, steps = 764\n",
      "10:29:29 [DEBUG] train episode 253: reward = 173.57, steps = 651\n",
      "10:29:48 [DEBUG] train episode 254: reward = 265.24, steps = 453\n",
      "10:30:32 [DEBUG] train episode 255: reward = 132.43, steps = 1000\n",
      "10:31:19 [DEBUG] train episode 256: reward = -21.95, steps = 1000\n",
      "10:32:04 [DEBUG] train episode 257: reward = -26.56, steps = 1000\n",
      "10:32:30 [DEBUG] train episode 258: reward = 188.48, steps = 577\n",
      "10:33:00 [DEBUG] train episode 259: reward = -106.38, steps = 660\n",
      "10:33:46 [DEBUG] train episode 260: reward = -34.32, steps = 1000\n",
      "10:34:18 [DEBUG] train episode 261: reward = 203.41, steps = 725\n",
      "10:34:36 [DEBUG] train episode 262: reward = 255.83, steps = 408\n",
      "10:35:22 [DEBUG] train episode 263: reward = -9.11, steps = 1000\n",
      "10:36:10 [DEBUG] train episode 264: reward = 39.38, steps = 1000\n",
      "10:36:46 [DEBUG] train episode 265: reward = 211.19, steps = 808\n",
      "10:37:04 [DEBUG] train episode 266: reward = 261.64, steps = 413\n",
      "10:37:48 [DEBUG] train episode 267: reward = 136.15, steps = 1000\n",
      "10:38:33 [DEBUG] train episode 268: reward = -15.21, steps = 1000\n",
      "10:39:05 [DEBUG] train episode 269: reward = 232.94, steps = 726\n",
      "10:39:22 [DEBUG] train episode 270: reward = 236.03, steps = 375\n",
      "10:39:50 [DEBUG] train episode 271: reward = 237.10, steps = 636\n",
      "10:40:06 [DEBUG] train episode 272: reward = 248.89, steps = 359\n",
      "10:40:22 [DEBUG] train episode 273: reward = 266.12, steps = 378\n",
      "10:41:07 [DEBUG] train episode 274: reward = 1.84, steps = 1000\n",
      "10:41:45 [DEBUG] train episode 275: reward = 157.79, steps = 825\n",
      "10:42:17 [DEBUG] train episode 276: reward = 189.86, steps = 719\n",
      "10:42:24 [DEBUG] train episode 277: reward = -35.46, steps = 155\n",
      "10:42:33 [DEBUG] train episode 278: reward = 3.49, steps = 211\n",
      "10:43:01 [DEBUG] train episode 279: reward = 238.85, steps = 641\n",
      "10:43:33 [DEBUG] train episode 280: reward = 180.11, steps = 713\n",
      "10:43:54 [DEBUG] train episode 281: reward = 232.95, steps = 470\n",
      "10:44:11 [DEBUG] train episode 282: reward = 257.30, steps = 383\n",
      "10:44:35 [DEBUG] train episode 283: reward = 192.55, steps = 533\n",
      "10:45:00 [DEBUG] train episode 284: reward = 256.59, steps = 578\n",
      "10:45:30 [DEBUG] train episode 285: reward = 253.76, steps = 650\n",
      "10:45:57 [DEBUG] train episode 286: reward = 215.79, steps = 626\n",
      "10:46:42 [DEBUG] train episode 287: reward = 127.63, steps = 1000\n",
      "10:47:10 [DEBUG] train episode 288: reward = 237.74, steps = 639\n",
      "10:47:24 [DEBUG] train episode 289: reward = 257.29, steps = 314\n",
      "10:47:44 [DEBUG] train episode 290: reward = 219.75, steps = 454\n",
      "10:48:27 [DEBUG] train episode 291: reward = -34.80, steps = 1000\n",
      "10:48:45 [DEBUG] train episode 292: reward = 225.38, steps = 415\n",
      "10:49:30 [DEBUG] train episode 293: reward = 132.93, steps = 1000\n",
      "10:49:42 [DEBUG] train episode 294: reward = 239.37, steps = 267\n",
      "10:50:08 [DEBUG] train episode 295: reward = 228.38, steps = 598\n",
      "10:50:26 [DEBUG] train episode 296: reward = 288.36, steps = 422\n",
      "10:51:11 [DEBUG] train episode 297: reward = 29.07, steps = 1000\n",
      "10:51:27 [DEBUG] train episode 298: reward = 233.57, steps = 359\n",
      "10:51:48 [DEBUG] train episode 299: reward = 205.36, steps = 478\n",
      "10:52:06 [DEBUG] train episode 300: reward = 233.38, steps = 414\n",
      "10:52:24 [DEBUG] train episode 301: reward = 214.90, steps = 402\n",
      "10:52:42 [DEBUG] train episode 302: reward = 216.78, steps = 420\n",
      "10:52:54 [DEBUG] train episode 303: reward = 272.85, steps = 274\n",
      "10:53:27 [DEBUG] train episode 304: reward = 273.54, steps = 724\n",
      "10:54:00 [DEBUG] train episode 305: reward = 201.24, steps = 734\n",
      "10:54:44 [DEBUG] train episode 306: reward = 51.44, steps = 1000\n",
      "10:54:56 [DEBUG] train episode 307: reward = 243.04, steps = 278\n",
      "10:55:11 [DEBUG] train episode 308: reward = 259.15, steps = 352\n",
      "10:55:25 [DEBUG] train episode 309: reward = 203.67, steps = 319\n",
      "10:56:04 [DEBUG] train episode 310: reward = 229.39, steps = 901\n",
      "10:56:25 [DEBUG] train episode 311: reward = 268.76, steps = 466\n",
      "10:57:09 [DEBUG] train episode 312: reward = 144.95, steps = 1000\n",
      "10:57:53 [DEBUG] train episode 313: reward = -7.47, steps = 1000\n",
      "10:58:37 [DEBUG] train episode 314: reward = 148.34, steps = 1000\n",
      "10:59:20 [DEBUG] train episode 315: reward = 143.54, steps = 1000\n",
      "11:00:07 [DEBUG] train episode 316: reward = 86.12, steps = 1000\n",
      "11:00:40 [DEBUG] train episode 317: reward = 205.33, steps = 744\n",
      "11:01:24 [DEBUG] train episode 318: reward = 162.79, steps = 1000\n",
      "11:01:47 [DEBUG] train episode 319: reward = 204.85, steps = 510\n",
      "11:02:03 [DEBUG] train episode 320: reward = 242.95, steps = 361\n",
      "11:02:15 [DEBUG] train episode 321: reward = 227.31, steps = 292\n",
      "11:03:00 [DEBUG] train episode 322: reward = 67.16, steps = 1000\n",
      "11:03:44 [DEBUG] train episode 323: reward = 164.35, steps = 1000\n",
      "11:04:03 [DEBUG] train episode 324: reward = 226.79, steps = 440\n",
      "11:04:17 [DEBUG] train episode 325: reward = 253.51, steps = 320\n",
      "11:04:51 [DEBUG] train episode 326: reward = 223.09, steps = 730\n",
      "11:05:07 [DEBUG] train episode 327: reward = 291.06, steps = 369\n",
      "11:05:27 [DEBUG] train episode 328: reward = 222.47, steps = 434\n",
      "11:05:52 [DEBUG] train episode 329: reward = 240.93, steps = 563\n",
      "11:06:05 [DEBUG] train episode 330: reward = 296.07, steps = 308\n",
      "11:06:20 [DEBUG] train episode 331: reward = 250.09, steps = 328\n",
      "11:06:37 [DEBUG] train episode 332: reward = 241.10, steps = 395\n",
      "11:06:56 [DEBUG] train episode 333: reward = 250.09, steps = 431\n",
      "11:07:14 [DEBUG] train episode 334: reward = 247.34, steps = 405\n",
      "11:07:15 [INFO] ==== test ====\n",
      "11:07:15 [DEBUG] test episode 0: reward = 271.18, steps = 299\n",
      "11:07:15 [DEBUG] test episode 1: reward = 261.38, steps = 254\n",
      "11:07:16 [DEBUG] test episode 2: reward = 0.23, steps = 271\n",
      "11:07:16 [DEBUG] test episode 3: reward = 248.13, steps = 267\n",
      "11:07:17 [DEBUG] test episode 4: reward = 224.61, steps = 330\n",
      "11:07:18 [DEBUG] test episode 5: reward = 216.57, steps = 342\n",
      "11:07:18 [DEBUG] test episode 6: reward = 248.86, steps = 355\n",
      "11:07:19 [DEBUG] test episode 7: reward = 229.78, steps = 296\n",
      "11:07:20 [DEBUG] test episode 8: reward = 274.99, steps = 322\n",
      "11:07:21 [DEBUG] test episode 9: reward = 181.28, steps = 491\n",
      "11:07:21 [DEBUG] test episode 10: reward = 247.33, steps = 311\n",
      "11:07:22 [DEBUG] test episode 11: reward = 235.35, steps = 499\n",
      "11:07:23 [DEBUG] test episode 12: reward = 247.33, steps = 311\n",
      "11:07:24 [DEBUG] test episode 13: reward = 278.01, steps = 264\n",
      "11:07:24 [DEBUG] test episode 14: reward = 239.55, steps = 322\n",
      "11:07:25 [DEBUG] test episode 15: reward = 254.86, steps = 298\n",
      "11:07:25 [DEBUG] test episode 16: reward = 246.03, steps = 364\n",
      "11:07:26 [DEBUG] test episode 17: reward = 250.75, steps = 338\n",
      "11:07:27 [DEBUG] test episode 18: reward = 255.87, steps = 402\n",
      "11:07:28 [DEBUG] test episode 19: reward = 248.91, steps = 298\n",
      "11:07:28 [DEBUG] test episode 20: reward = 287.79, steps = 324\n",
      "11:07:29 [DEBUG] test episode 21: reward = 275.28, steps = 355\n",
      "11:07:29 [DEBUG] test episode 22: reward = 236.50, steps = 367\n",
      "11:07:31 [DEBUG] test episode 23: reward = 192.14, steps = 454\n",
      "11:07:33 [DEBUG] test episode 24: reward = 169.51, steps = 788\n",
      "11:07:34 [DEBUG] test episode 25: reward = 261.63, steps = 444\n",
      "11:07:35 [DEBUG] test episode 26: reward = 228.02, steps = 292\n",
      "11:07:36 [DEBUG] test episode 27: reward = 239.05, steps = 375\n",
      "11:07:36 [DEBUG] test episode 28: reward = 240.58, steps = 326\n",
      "11:07:39 [DEBUG] test episode 29: reward = 161.61, steps = 1000\n",
      "11:07:39 [DEBUG] test episode 30: reward = 252.69, steps = 290\n",
      "11:07:40 [DEBUG] test episode 31: reward = 258.46, steps = 288\n",
      "11:07:40 [DEBUG] test episode 32: reward = 273.27, steps = 326\n",
      "11:07:41 [DEBUG] test episode 33: reward = 256.26, steps = 282\n",
      "11:07:41 [DEBUG] test episode 34: reward = 267.48, steps = 331\n",
      "11:07:42 [DEBUG] test episode 35: reward = 244.72, steps = 344\n",
      "11:07:43 [DEBUG] test episode 36: reward = 247.76, steps = 336\n",
      "11:07:44 [DEBUG] test episode 37: reward = 217.31, steps = 372\n",
      "11:07:44 [DEBUG] test episode 38: reward = 247.06, steps = 342\n",
      "11:07:45 [DEBUG] test episode 39: reward = 211.10, steps = 340\n",
      "11:07:45 [DEBUG] test episode 40: reward = 264.69, steps = 312\n",
      "11:07:46 [DEBUG] test episode 41: reward = 232.47, steps = 303\n",
      "11:07:47 [DEBUG] test episode 42: reward = 228.02, steps = 382\n",
      "11:07:48 [DEBUG] test episode 43: reward = 251.82, steps = 327\n",
      "11:07:48 [DEBUG] test episode 44: reward = 252.08, steps = 304\n",
      "11:07:49 [DEBUG] test episode 45: reward = 220.72, steps = 321\n",
      "11:07:49 [DEBUG] test episode 46: reward = 238.09, steps = 409\n",
      "11:07:50 [DEBUG] test episode 47: reward = 230.84, steps = 334\n",
      "11:07:52 [DEBUG] test episode 48: reward = 215.48, steps = 612\n",
      "11:07:52 [DEBUG] test episode 49: reward = 223.35, steps = 372\n",
      "11:07:55 [DEBUG] test episode 50: reward = -17.11, steps = 1000\n",
      "11:07:55 [DEBUG] test episode 51: reward = 228.41, steps = 294\n",
      "11:07:56 [DEBUG] test episode 52: reward = 255.20, steps = 339\n",
      "11:07:57 [DEBUG] test episode 53: reward = 220.52, steps = 498\n",
      "11:07:58 [DEBUG] test episode 54: reward = 215.52, steps = 291\n",
      "11:07:59 [DEBUG] test episode 55: reward = 243.97, steps = 491\n",
      "11:08:00 [DEBUG] test episode 56: reward = 211.23, steps = 440\n",
      "11:08:01 [DEBUG] test episode 57: reward = 213.63, steps = 306\n",
      "11:08:01 [DEBUG] test episode 58: reward = 252.84, steps = 302\n",
      "11:08:02 [DEBUG] test episode 59: reward = 284.78, steps = 265\n",
      "11:08:02 [DEBUG] test episode 60: reward = 209.75, steps = 399\n",
      "11:08:03 [DEBUG] test episode 61: reward = 254.63, steps = 268\n",
      "11:08:07 [DEBUG] test episode 62: reward = -49.69, steps = 1000\n",
      "11:08:07 [DEBUG] test episode 63: reward = 278.15, steps = 308\n",
      "11:08:08 [DEBUG] test episode 64: reward = 243.76, steps = 468\n",
      "11:08:09 [DEBUG] test episode 65: reward = 203.84, steps = 374\n",
      "11:08:10 [DEBUG] test episode 66: reward = 267.18, steps = 285\n",
      "11:08:10 [DEBUG] test episode 67: reward = 233.67, steps = 420\n",
      "11:08:11 [DEBUG] test episode 68: reward = 251.62, steps = 337\n",
      "11:08:12 [DEBUG] test episode 69: reward = 237.55, steps = 330\n",
      "11:08:12 [DEBUG] test episode 70: reward = 282.45, steps = 364\n",
      "11:08:13 [DEBUG] test episode 71: reward = 271.30, steps = 328\n",
      "11:08:14 [DEBUG] test episode 72: reward = 266.89, steps = 313\n",
      "11:08:14 [DEBUG] test episode 73: reward = 224.90, steps = 364\n",
      "11:08:15 [DEBUG] test episode 74: reward = 231.54, steps = 401\n",
      "11:08:16 [DEBUG] test episode 75: reward = 240.63, steps = 249\n",
      "11:08:16 [DEBUG] test episode 76: reward = 238.24, steps = 325\n",
      "11:08:17 [DEBUG] test episode 77: reward = 267.44, steps = 272\n",
      "11:08:18 [DEBUG] test episode 78: reward = 262.40, steps = 370\n",
      "11:08:18 [DEBUG] test episode 79: reward = 222.95, steps = 283\n",
      "11:08:19 [DEBUG] test episode 80: reward = 231.19, steps = 319\n",
      "11:08:20 [DEBUG] test episode 81: reward = 209.95, steps = 433\n",
      "11:08:20 [DEBUG] test episode 82: reward = 224.54, steps = 323\n",
      "11:08:21 [DEBUG] test episode 83: reward = 198.86, steps = 316\n",
      "11:08:21 [DEBUG] test episode 84: reward = 233.75, steps = 260\n",
      "11:08:22 [DEBUG] test episode 85: reward = 217.96, steps = 346\n",
      "11:08:23 [DEBUG] test episode 86: reward = 259.67, steps = 307\n",
      "11:08:23 [DEBUG] test episode 87: reward = 209.53, steps = 406\n",
      "11:08:24 [DEBUG] test episode 88: reward = 255.52, steps = 316\n",
      "11:08:24 [DEBUG] test episode 89: reward = 268.38, steps = 247\n",
      "11:08:25 [DEBUG] test episode 90: reward = 249.62, steps = 336\n",
      "11:08:26 [DEBUG] test episode 91: reward = 224.22, steps = 364\n",
      "11:08:26 [DEBUG] test episode 92: reward = 284.50, steps = 305\n",
      "11:08:27 [DEBUG] test episode 93: reward = 232.68, steps = 283\n",
      "11:08:28 [DEBUG] test episode 94: reward = 227.78, steps = 326\n",
      "11:08:28 [DEBUG] test episode 95: reward = 252.30, steps = 280\n",
      "11:08:29 [DEBUG] test episode 96: reward = 214.03, steps = 347\n",
      "11:08:31 [DEBUG] test episode 97: reward = 165.31, steps = 765\n",
      "11:08:31 [DEBUG] test episode 98: reward = 254.61, steps = 356\n",
      "11:08:32 [DEBUG] test episode 99: reward = 281.94, steps = 407\n",
      "11:08:32 [INFO] average episode reward = 232.31 ± 51.68\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAD4CAYAAAD2FnFTAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABkQUlEQVR4nO29ebwkVXn//3mqqrtv333uzJ1hVmYYhn1n2MGAIKCo4xKVmKhR8yMakojfGCNqokZREhNNNJGEBBWNSsDEgAvIIrvAOAgDs8IMMwOz39nu3ktVnd8fVafq1KlTvdzue7v73vN+ve7rdldXV52q7n6e86yHGGPQaDQajUbGaPQANBqNRtOcaAWh0Wg0GiVaQWg0Go1GiVYQGo1Go1GiFYRGo9FolFiNHkClzJkzhy1durTRw9BoNJqW4tlnnz3AGOufyHtbRkEsXboUa9asafQwNBqNpqUgoh0Tfa92MWk0Go1GiVYQGo1Go1GiFYRGo9FolGgFodFoNBolNSsIImojotVEtJaI1hPRF/ztfUT0ABG97P+fJbznRiLaQkSbieiqWseg0Wg0mvpTDwsiD+D1jLHTAZwB4GoiOh/ApwA8xBhbAeAh/zmI6CQA1wI4GcDVAL5FRGYdxqHRaDSaOlKzgmAeI/7TlP/HAKwCcLu//XYAb/MfrwJwB2MszxjbBmALgHNrHYdGo9Fo6ktdYhBEZBLR8wD2A3iAMfYMgHmMsT0A4P+f6+++EMBrwtt3+ttUx72OiNYQ0ZqBgYF6DFWj0Wiamv97bheGc8VGDwNAnRQEY8xhjJ0BYBGAc4nolBK7k+oQCce9lTG2kjG2sr9/QoWAGo1G0/SMFWwUbBevHRrDDf/9PH66dk+jhwSgzllMjLEjAB6BF1vYR0TzAcD/v9/fbSeAxcLbFgHYXc9xaDQaTS08uGEfRvL2lJ3v/betxpd/sRGD457lcGg0P2XnLkU9spj6iajXf5wFcAWATQDuAfABf7cPALjbf3wPgGuJKENEywCsALC61nFoNBpNPXh53zD+6Htr8NmfvDhl53z10BheOzSGIV9BHB5rDhdTPXoxzQdwu5+JZAC4kzH2MyJ6CsCdRPRhAK8CeBcAMMbWE9GdADYAsAFczxhz6jAOjUajqZmBEW/2vmcwN2XnHM3bGMnbGMpxBVGYsnOXomYFwRh7AcCZiu0HAVye8J6bANxU67k1Gk1l3P38LqzZfhh/eNFSLO/vbOhYckUHGcsAkSoc2XjGC958tT09Ndn3rsswWnAwWrAxNO65tQ6PNoeC0JXUGs0M4Lu/3o7vP70DX7v/pZL7/cF/PoOL/+5X+OB3VuO7T26b8Pn2D+Vw74vxQOv+4RxO+Ov78P2nJ9xgdNIZCxREfP782qEx3PbENry4c7B+5yt65xvNO4IF0RwuJq0gNJoZAPPzBPO2W3K/J7YcwM7D43h48wA+/9MNEz7fXc/uxJ/88LcoOtHzbRsYBQDc/Xzz5qWMFbxZvMqC+Pw96/HFn23A1x+MK1rbcbHryHjV5xv1g+GjeTuIQRxpEheTVhAazTRkOFfErzbtC55zxeC4pRVEvSjYLhgDHDeawT7uz5bbUgYcl2HT3qGaz/XjZ3di/3D94gVjJVxMB3zXz6giw+mLP9uAi27+VdXCfURUEDnfxaQtCI1GMxmM5m2c+vn78aHvrsGeQW9Gm7c9oecoK47qj+ubLPw/55AvYDOWifvX78Wb/vnxioR7rujghZ1HYts37x3GJ+5ai8/+ZF3tg/bhCiKrcDEN+zP80UJcQdyzdrc/1uqUcGBBFJwgzXUoV4wp10agFYRGM814auvB4PGwPyPN+0LLrULozOnMAPBcLl/+xcbA9VIJXLjJQu7gCFcQBg6MFuAyBIHZUvzFXWvx1n95Mha8fXGXFwtIWcmibChXRK7ogEnKynFZoLBE+HVmFMfkMYKxvKdEckUHX/jpeuwfzgX3WnarlUOst9jrZ04xhkBZNBKtIDSaaYYoWHhGTuhiCoXkaN7GzfduQq6ozjKf05kGANzyyFbc+tgr+O/fvKbcT4UTWBDR7QeEArCcP7aCIi5SdFx85PvPYp2vALjSK0jCd8Nuz0V1bInMrNM+fz/eecuvcek/PIL/99/PBwL8s/+3Dmd98YHAuuKM+sJftn4ABC4gbkH85Lld+M6T2/Fvj7wC27/YcnEekVzRwf6h8J7sHQqtqWZIddUKQqOZIE9uOYCv/nJTo4cRQ5yRjgUKgruYQqF3yyNb8W+Pbo0I/t72VPC4r8NTENsPjgEAerLha+XglopssRwYLgRj5IpJNePevHcY963fi7/88Qv+dXjXJCuTjXs8BWEa6pRZfuz1u4ew4+AY/ve5Xbh33V4AwI9Wv+qNaSQqiLmVIFs/uaITnJ9bEC/tGwYAzBLum0rhJfG5u9fjhv9+Pni++8g42lKeWD44UgiO3yi0gtBoJsj96/fiO09ub/QwYoiN3rgQVrmYDvkzVLEcYbavFACgLeUFaff4mTmqtM8kuMx3pFn4Qd+CGM7ZQcBatgqAUMl1ZSz/OtzYvq7LsMFXEEn++t2KrKLXDo1Fnh8Yjra1GPGtBHnsXHHMak9htGCDMYatflaWuKfqepJYvyeaLpu33aBO5aafb8CVX38MN/7vi1UpnXqiFYRGM0EcxqpyJ0wGX/7FRnzl3o2RbdwXDnhZQ67LAqElCj3u4uGKAADSVviYC11eUWxXkQGVFKTmMYiRfKggiop7yIV0RyaaSSQKyhd3DQbuNJU7CAitH5EDI/mIW+nASFRB8PvnSBF9vn1edxtc5imt9b4LbEhw6+UTXHaMMTz+8kBESb92KK7Azlnah6Nnt2OtX2vx0MZ9ODLeGHeTVhAazQRxXE+I2lUGJevJb3ccxnM7jkS2DeWiLiZ51s0JZvCC0BXTYLmC2O1nQtlVpEAFCkK6NVwYD+eKgVWQV9w/7uPvbIu6tcSxPrx5f2D92K4szIvYeXgMOw6Oxo59YKSArfvD7QOCBbFm+yFsGfCWt4lZEL4SmN/TBgDYdWQcB/0gtxg7SLIgHt68H++7bTVue8IrQBzKFZWB6O5sCu86exEA4CvvOBUPfPx3MLerTXnMyaYevZg0mhkJF7YFx4VlNmauZbsMBsWFY1fGwrA/S88LaZei0BsPKnhDheK4DKZBcFwW/PG3VOM6CbKYhPMxxkILIhfGIFTuE+5i6pQtCH8MP127G3esfg1nLu7F+t1DsVjH797yFDbvG8aHLloW2Z4yCQeG83h5f+jbFy2I3/23p2LXwOGK96ieLIBoMdu2A6HCSXIH8WytJ7YcwP/3umOwU2E9AMDiWVlcefJRKDgMbz9zYcTCm2q0BaHRTBAu/PJV5r3XE5exmCAbztno7/ZSVHMFJ+JOEWU8d+PICuKaU+fj/GP64LgsMjOuJn0ztCDCsRUdz9WVMsnrPZRPTgsNXExS3KNgu/jBMzvwZz96Dh0ZE5+48nhYvkIT2ewHd2UL4ujZHTgwko+4hOQgNWfHwTG87u8fxj1rd+Oj//VsoBC4BSHO/itREDw2xGMgrx2Ou78A4KQF3ejJpvD/3nBcQ5UDoC0IjWbCuBNIa6w3quDscK6IuV0ZvDIwirGCExmfKLC5e2S0ICgQ5lkQlmFg3HEwJigPVayg3LjE2ADf1pNN48BIPpi5qwQq9/dn02ZkzAXbxYMb9uGYOR2474bXIWUaMAyKuZg4shBeOrsDa3YcCu7JnM5M4GKS6yQefclbxfLPf/QcAC/2AABH+QpiSEgGGBPuYZKlxRXRzsPjKDpuLFjOOXZuY5spimgLQqOZIIEFYauDklMyBjduQYzkbfRm00hbBsaLTqTOQXT58Owd0YJwXS9l1PBn5aLgPTJexK+3HqhwXOH45HPzVNqBEgqCC1+XMYwIBXp524XtMvS0p5Dy3XqmQYlB6p2Ho26cZXPacWSsGNQ6LJqVxcBIHj9duxv/89tdJa/p6Ve8WoyjfEUxmNAOI2nCwDO4Co6L7QdGsevIODoU7TwyVmOtBhGtIDSaCcKFX6NSEPkY4gFaG11tFrIpE7liaEG0pYxgNp4rOhj2FYNYN2G7LkwimBRXPv/04Mt47388g5dL5Oa/MjCCK772aGAdiEPjx+r16yl4TYTKxcTdN44bFcQFx4XjMlhC3YPKxcQZKzjICm6aJbM7AITprwt623BgOI8/+9Fz+MRdawEAn73mRCyalY0J7017h2EaFFSYD/oxhT4hNRhI/j4cFFxZ40WvrUZve1q5b7OgXUwazQRxWRO4mFgYROZ4CiKFbMrEWMEOLJz2tBXM4sXAbDQGAZgmwTQMpXUCABv2DGHFvC7leF7aN4It+0fKuJg8BcGD5Kr7FyoIN+LrL9iegjCE4g2D4goimzKD4y/obcPWgVF0ZizM7fKE+64j48hYBuZ0ZnBw9GDkvRnLgGmQclzdbRY6/doMPq6+jjQOjRYwtyuD/cP5ihRE0XEx4ivyRz5xKYi8Yr7jj1Lf10ahFYRGM0EctzlcTLIQHsnb6GyzkE2bGC+6QRA9mzLhuAwX/92v8IaT5gXv4e4W7/2+BWGorRMAeGUgnjrK4WPhQjLiYuIKol1KXS1nQSgURCYVOj9MhQXRnhYVRDZQELx9yG5fQaRMI5amnLFMmKSOaxw3rwvtfmYVd4P1+VbA/N6spyASYhAHR/OBMik6/ueUsbB0jmfVHO1bN82EVhAazQThcqCRWUzyLJ+7i7p9F9O4EKRuT5sYyhWxbygfEfJiZ1Ke5moZBhxFhhSASIqoajxAKPS5wrjp5xswy3fFzJZcMkU7fg6xAC6qIBw4LGpBmAbFahayaRPwL3HRLC8ttSNjosOf/R8aKyCTMmEpAtyZlBFr3XH9ZcsxuyODd61cBMswImOc1eEpPG6dJFoQowUs6Mn6CsLFcM4OFFazUnMMgogWE9HDRLSRiNYT0cf87X1E9AARvez/nyW850Yi2kJEm4noqlrHoNE0gsDF1MBCOVea5fNUyq7AghBdTGYgvPi27jZLWQcRBqnj17Zpb7KC4PeExxVc18sG+o/Ht+Hv79sMALGir4ITt8B4zYDjskiQXRWDUFkQYtxhgV+30Om73QBPuGcsLwNKDnBnLDOmIC5Z0Y8PXbwMXW0ptKUMGBStnga8jCiD4gpi3a5BDAzncWSsGGRA2dyCaKu8v1UjqEeQ2gbwF4yxEwGcD+B6IjoJwKcAPMQYWwHgIf85/NeuBXAygKsBfIuImids38Js3juMX67f2+hhzBiC7p0NtCBsyYLg6aE8BhG1ICxBQXj/e9pTGMmr0lw9oavqrrH9QLKLiY+Fp8Q6jOHbfuUwT9/0MpBCAVxUVGhz4Wu7LFLBzV1MogA3FTGI9kzoHFnQ6ymIrowVKAjGvFhDpRaEmHpKROhIW4EFwduTHD27HWnLiLmY3vzNJ3DpVx8GEKbIFnwLojPT3E6cmhUEY2wPY+y3/uNhABsBLASwCsDt/m63A3ib/3gVgDsYY3nG2DYAWwCcW+s4NN66w5/9v/otnKIpTVgH0bgYhMuirT644M+mzCAGwWfg7WkzEMZcqfVkU+o0V0q2IFzFSnEc2cXEYyJAWGeQMgk92dC1Is+4xd5RrstQFMagUhAqKyArxCi4gujMWGgTMpMylgmDKBbk50FqzupPXx5kLgXHT5uBMv7985bgr998Ej500TKkTSPSi4lfM6814SmyngVRRFfbNFcQIkS0FMCZAJ4BMI8xtgfwlAiAuf5uCwGIjeV3+ttUx7uOiNYQ0ZqBgYF6DnVaYjtuQ/sCzTTCIHXzxCB4LyXTIN+CsEOlkTYDwRu6mFIYLzrBMXiaK7cgkhRBUuO+0MXk/WeMxdZJMIgiTfjk+yfGExwWtSDygYspFF2WQbE+UaLQX9DrCeUOwYIAPEvBMuJtwttSZiTGkVFUM6dMI2hBnk2Z+PDFy5C2DKQtM2JByNYEVxBefYo7/S0IDhF1AvgfADcwxkotNKtq3K78FjLGbmWMrWSMrezv76/HMFsSx2Ul2xzYflWmI5nj04kb7ngO//dc6UKmqcZpgjRXW4pB8M/f4gpC6MUkrrEcuJj8lFPevtpl3oycVycnVSgnWxDx/bjS4t9hyzAiglr+bsuZT+LrQZqrEU1zdRnz22+PxI7R15FGR9pEd9ZCygyVAo9ByHDXE0elRHgRIhBdiyJjGZHvg2wdzfNdTLxtx4xQEESUgqccfsAY+19/8z4imu+/Ph/Afn/7TgCLhbcvArC7HuOYrnztgc249tanE1//6Qu7ccXXHsVQrpj4g251Htq0H2t2HGr0MCIEzfoaqCBcYZb/1NaDwRKapkHIpk3sG8rjpl947cDF9RxkBTGWD60Iy49BuIwlLlGqihsA8Q6oogXA75NpIMgmErff8shWfO3+zTEFwb/TPMjuMHWQ+rYntuHyf3wU63YNBuMwyBPC//Les4LGfVw5ZSxTKfwzlhlRHKrFiFImBd1oU0KjxoxlRL4P8neDWxD8c+qc7i4mIiIAtwHYyBj7mvDSPQA+4D/+AIC7he3XElGGiJYBWAFgda3jmM7sPpJTLnzCOTRaRN72gl7NsND5ZFDK3dEomqLVhu/CydsO/uC2Z/CDZ7xV0iyTIoFgILrGMveTdwsWBL8e06BA6HLhLK/PnPRZyAqFMShdTKI1w4XoE1sG8NjLByKTHEeIsfAgu+3E01xtl+HBjfsAAEfGisH4urMpEBEuO2EuFve1A0AQh5BjDeJ9KmdBiEpBPEZaUhCiNWEZhNl+Wuthvzq8q8ktiHqM7iIA7wPwIhE972/7NICbAdxJRB8G8CqAdwEAY2w9Ed0JYAO8DKjrGWON+4W1AOWEIzfh87YbCehNJ2yn+dxnQZC6wXUQgLf2tOOyoH2GaRixxXLEWTEXXLxbar7oBhlLooLgx29LmRFhlxTrkr+n4jG44LRMinRp5S4k22Gx7rSOw1BwBAsiIc3VZSxohtfZZgXfFdUyqYEFIcUaOG2paJqr2oIwhMeSgnDUFsTsznTwvsMtYkHUPDrG2BNQxxUA4PKE99wE4KZazz1TcBhLbEYGCAVbtgvGPMGl8q22MrbrNrEF0XgFEaw97VsGlkH4+BXHoSebwo+f3QnASwflBArCDxbnbCcIPHuV1FEFkU2ZkYK1JFem/D31LBw38h6DKKhGBqIZT7Ggu29BWAYFs3OHRb/fPM2Vd2UVj8EX3hEJXUwGLFNhQaSMQHFYBoEUSiSdZEGYkotJUBazOzLB+w7PpBiEZnIRM0FUhO0NPOEw3eIQrusFT5vtukLF3BgDmAeVAUFBBH5+wkkLuvEP7zo92F81aWhPsCB4mmugIKTGdeXSXMUxykt3mgZhoZ96CoSz7KJCQfBCQMukQPgqLQhXbM/B4DCGa06bjz99/YrYGLmLKW0ZSgtCdDGprAcASFnhdtGaiLmYilELgiskriCaPc21uUenAVDexRQEAYWZ2HSCK4Zmu65GBKnHCjae3XEYl6zoj9yPcYUFIWMqhCG3IPKiBcEL5VhyDCIpqy4WpHbjit00CNdfdixmtafx4MZ9QTaQ47peXYfgJrX9LKaUYQTum1ihnEERJc2tTdX1AmGNhBxrAAAizwrgyjSVsFKgnGbLSVtGpDuuWCU+pzMT7MtjEJ2Z6V9JrZlkHDce/Iu8LjVIm25xCFtKk2wWGuFi+tnaPXjfbatxcCQfEcY8J1+0IDhP33g57v/466CSddzFkSu6YeaP0GqDx7f4ymbcRVJpkFrV8M8kQlvKxIcuXoZsKmz/YTssVr3t+llQKcsIXUyKQrl45pOrVJJANItJtqoylgHy60CAEhaEKSoIwYIwky2Ivo40iLzkgVaJQWgF0QIwxmIzMxF5ZTPZpG91mt2CmEoFwRvrjRediCAd82fhucCCCH/aR/W04bh5XaVdTHZ4PJ7mCiAIEHOhymMHyfUR0edMqvQGokI3ZRqB4ncCV2K4vyjsMwkWhGVQZNlQ2/XcWknCvU2MQcQUhPcav1dJSiYtuJgsOUgtKgjh2nmKa8o0wtTdBi8pWo7mVl8aAEjsqsmxJVdHs/nqa4W70JrtusJeTFMXg+DCtOhEXTE5OQahCL6WdjG5sSC1t907Lo9BtKdMHEExllGWtx187EfPIyWnwyq+u0lpocogtctQdBhSphGJQUQsCCLsEtLAHcebUKkC0ICYxRRPc+WuNH6vKrEg5OtRFcp99poT8e5zvPKvcoV6zYRWEC1AuRiE3IM/qQ1Cq8Kvp9ksCLnvkMzgWBF3PfsaPnzxMmUmzETgBWpFx41aEIVogoIyBqHYxgvWNuweCpbn5GmuQPidavP99rwJnvwd23Mkh/vW78W87mjPIqWLKUFB8MaDkVYbrp/FZEazmMxIHUT0mvhxVAFoQKyDiHdt5dYFv39JMYhomqtUKKfojXXp8f3o9ju3pn0l1Nbk1gOgFURLwJjXII0xphQ0YR8d//90czEFFkRzKb6g3XdCHcRDm/bhSz/fiCtOnBcsClMr3ILggpIzJlkxKmWgEpi8HuG7v94eeS8XwIGC8F0vfBlOWejLcTAOY3HFLruYuBuLL34kfn8d5lkQQZqr0sUUL+KzpUwnETHNVbaquAXBZ/aVWBDiLrxZ33CuiKLDgvuRNkNlwMfblmp+D3/zj1AT/MAqTS1stpl2rZS7/kZRbkW5oACsjuMWjykqzHFh0R+gcgtCrEfgGCoLgruYfIUiT0KSMrrKWRBeawpu/bjKNNei4wYuJp6tJV6f7KZxGI9BqMWbqCBiQepU1MWUpGR4cVzKjNZJcCV2/Q+fw1lffAD7hnLB9uC9fvyiFSwIrSBaAD47SwpUy4Kz2WbatTIZgrYelFuTuujUX7HZZVxMHKUFodiWTZmQDQtLVBCOZEH4CiU2KUm4Fyr3mzhrT5kUuU+ukJDB12oI6iCsUEFEC+Wix3dct3QMIh1WUicFqXkMJ+kY3IKQrRfuBtuwexAA8NVfbvaPKygIbkFYWkFo6gDvKZ8k92XF0WyCtFaa34Io3XainuPmArcouZjGJQUhCy5AHaTm2UEiBkUtCIPCWS+3IORU6qREAlVqciwG4UgxCP8YacsIurlafh3EuKLOQ7YUbMdTKokxiBIWRFtKDlKXjkHICiZtmnAZsLy/M7rdiscstItJUxcCQZNkQUjm/nSLQfAZZrNdl9xjSCZYdrNEinK1iEWR4udeiQUhx1tNv41ERprJWpKLyTKMoP4hsCBkF1PCNaruTbQ1hRkoBV4Hwa8xbRlhHYRvQXAFFC2Uix6fH6+SGESiBVEuzTXBwuDPx6WYkKgg+D6qdSaaDa0gWgD+W0yqb4hXrzaXIK2VZrUg+HCSYhBy8kA9EBvbRSyIomxBlA9ScyEoz2TFLKa87cI0KJj1BjEINx5rUMGtK7GhnTgObpkU/DRbl4XKJu3XC3h1EAYyCamlsjLkLTuSAszZNK+kNmNWFW/uFyiIMi4m2cLgilRU2AZFP4/QgtAKQlMHWJkYhFy9Ou1iEEGzt+a6rnIuplIFfrmig4tu/hUe3rQ/9lopxDRXVasNwGsXoYo3yAIzzMePCipDyGKKKwh1FlM5C0I8R6SwzD9uUQhQ24KLyeV1EJYBq1IFoagmF0mqg/jgRUtx4xtPiLw3MUjtWwRyS3X+fExot5H2q7Plfdqs5he/zT9CTdkZtPxjbTZXTK00qwXhlElzFSuEZYbGi9h1ZDxYAa1SgjRXSUGMCVlMSUJNFphmoCCiYsAyKBDiBYcriGgMIpYYkfCd4/EF8RzRILWvIGzXi0GwsL1H2jKCbrApQUnJ1yJbAap2IyIrl/bhbWcswAlHdUX2Oaa/E3P9ame+vVyaq2xhcMUhph2nJCtDWxCaulI2zXWau5iaNoupTKFcqc+tIFREV0PYl4pJCiIUSElCTXYxcUUiCyqDKNi3YDuwBOHMYxAb9gzhuVcPB+9Jsm6DOgBBQRgKd0vBCVvV89uZNg04DkPRZrEFkESlIFtL3OWXpCjndGbwT9eeifa0FWvZIR9fFez3xq1+nSsD8fOQA/qWDlJr6gn/7VXqYio2mSCtlWa2IIjCal8ZLvxV7pewQV11brOCzQP2buS4OWHGmiTU4hZE2NVUxDLLxyD+/dFX8Nn/Wxe8J8n7V1RYEFF/vB/U9QWqaEFkfAui6LqwTCOiZEpZEIUyFoRI0nEqjUHISkiMqXBkFyQPcGsLQlMXnCDNtTIXk9NkvvpaaaZeTNsPjOINX3sUB0byYCz0Z6viEI6bbPkEFkSV1yR2trXrZEFkpJmsmOZalFxMHcK6EGJgvJwFIQpDlQXBj8VYqFi9NFfv8y/pYjInriCMBEukbAwiCFLLMYhwjN1+p1b51oSV1DNEQRDRt4loPxGtE7b1EdEDRPSy/3+W8NqNRLSFiDYT0VX1GMN0xi0zg44FqadZDKKZurlu3jeMl/eP4JWBUQBh0FalIAILQqUgJmhBhDGIZBdTpTGIIN1SClLLrTZMv80FEPZiEscCJE9K8goXkyqjR7SA+HHDNFfPgpAXCQoeJ8Qgku6DiGghqI6fHIOgyPjl6wGA2Z3RvlTyOWdSkPq7AK6Wtn0KwEOMsRUAHvKfg4hOAnAtgJP993yLiJpflTYQ7kpIbLFcplCu6Lj4wTM7GrbyWa3YzsSE6WTAhRef8WYDBRG/t6WaDE60826kWZ9iPQggWajJ9QJhDCIepOaz8rzttdqe05mBQcDcrlDoRXomJXw0YRaT2Lso7mLKCYH+sH+RAdtxUXTDOgjVNfLH3gpx4WeRVOQmkhTLCF1M6mOkEy2I8HlfR1r5Xv6xzZg6CMbYYwAOSZtXAbjdf3w7gLcJ2+9gjOUZY9sAbAFwbj3GMV0JXEwVt9qIPr/lka34zE/W4Z7nd0/OACeZZrIguFDkM972lDejVhWElaqDCBZ3mqAFYTtuRECPF8tbEEl1ELE0V4paEIZBuHD5bDz2yctw9Oz22FiA5M8mzGIKzyE36wOiabqFiAUBf01qQ1qkR+0OsgyjKguifJC6tIspnuYajjFJQfCJw4xxMSUwjzG2BwD8/3P97QsBvCbst9PfFoOIriOiNUS0ZmBgYBKH2txw673yZn1RoXPfur0AwgBjqyEvet9IuPDKxSyI6oLU+UDQV2tBhC6maJA6PL9qLQhAVQeRHKQOFgzyLQgiwqJZ7ZFjiEqxXB2EaKWo0lxFBRdmPplBZbWXxaS2QoK2GH7shL+/krUW5HUl5O3Ja1KrezFFXEwJCoLrVZ3FpEZ1x5XfLsbYrYyxlYyxlf39/ZM8rOal2m6uotBxXYYNe4a87S0avObX0wwWROBi8me8QZBaUQtRyoIoBi6m6j4TO6FQTiQxiymxklqR5iooCDOy/nL4WEzRTVLecpqrXMSXUrSmCCwI0+/F5HrdXMXZuhg74Mcj8mb8E7UgVI8TLYiELCdRQcxKVBDRBojNzGQqiH1ENB8A/P+8ZHQngMXCfosAtKbvY4rgs7OKFYTwnC8CAyQXdDU7tiBoWcJMdarggj1wMZWIQQS9mOpYBxFp1pfwfUjMYkoMUisK5YQ0V9mdw/WM6GJKyrCTXUyykrIUQWpRqfD1IFKmnMUUdzcZfuwkjEFM3MUULDmatGBQYEEkxyA6M2qLnX/kM93FdA+AD/iPPwDgbmH7tUSUIaJlAFYAWD2J42h5AgWRFIOQtj+6eQCrt3khIVFwtWyQWlwfoMFGBB/LuK9suYtJFYMoZflMNItJTPlNtiAqczElVVKbhmBBOG5csfjPRYWdGIOQgtTyGNKKGIRYOxE23ovGIFQ1CwZ5io1PhKpVEIZCWZRPc012McnxCU5gQbSAi6kuTmki+hGASwHMIaKdAD4H4GYAdxLRhwG8CuBdAMAYW09EdwLYAMAGcD1jrDUl1xTBf3xJszR5+33r9+K+9Xux/eZrIr7xpJ5BzY4oRG3XhWk0buZVkLKYSqW5lgquB0HqqrOYhFYbCROG6usg4r2YrISZNT9+mE3FkLaobB1EkoLghWXKLCYrKmwjldSK2IFXAR5+RhUpCEUsQ3xcPs012cWUMg1cf9lynDS/J7IP/z60ggVRFwXBGPu9hJcuT9j/JgA31ePcMwEuQ5L8vKWCt6IboHUVhJhOObUmxI6Do+jIWJjj57QX/UrmfKAgvJ9QyTRXVSW1lLr72EsD+Pw96/GLj11SUnAEgrmEi6laC2LRrCzSZrgug2WQMmAbHt8AEGZh8XUb1OP1FUQqwcVklAhSi1lLpmRBiArM5AoCEQuiHjGIpDWp+dhkF5Q85r+86oTYe/m9qkSBNZrmt3E0gYWQZEGUEpqRkv9iaxpqogKsxGe/4+BobAGdifLH338WX71vszCWqAXRVrKSuhIXk/fapr1DeOXAKAbHiyXHU3RCwVxtDCJJOL/ltAVY89dXBNsNosQCsqTxlGv3HVgQ0ow7rYpBCJXU4hiS6iBEC8I0qToLooyCKNusLyGu412b+r38+1yJAms0WkG0AOWWHE1KMRwvOBGBmmtZCyKab//0KwcTlSVjDG/+xhO4/antdTn34HgRQ7lQaBekLKbAxaRIACglPPOSi4krjHKJBPxeFF2W+LknZTHJmwPfvUHoEgKqllBJLe7HEa+H34+kscQUhKSkQheTOkgd7FeqkloQ5pZhTLwXk3B/giB12TTX0i4mFfy7W0kabqPRCqIFKJfFlORiOjCSR8ERgtTTwILYvHcY1976NB7fckC5b9FhGM7bODiSr8u5i44bscK4iykWg1AEm0sFqYuSi4mfI1cmkUB0MSXVUCRXUqtjEAAi6xUYQpBa9T5HYdGVdTFJK7WFY1C4mBw3SFkN9pOymFTKgsh7PNEsJlVmVKIFkZDmKruYVGgLQlNXyhXKJc2m9w/ngu6fQAvHIIRagcNjBQDeegoqwkK25Gv9zpPbcOea1xJfF8nbbqSddzE4vlQop1C+lQSpuZDPV2BBML+zKR9HkkWZ1IE0qQ4itl+ZILV4XruMiykIUvO1nivJYrLdoOiNkzKi3VxVhXJyFlOSJSWSFKTmx0+yAkIXk5TFZJV3MX357afi/GP6cPxRXWXH12i0gmgBnHJ1EAmCYmA4Pz2C1MJ1c0GSdC3BTLyEtXTnmp0Vtx0p2G7kHsoxiNJZTCWC1IGLyY28v5QF4bgs6ONTdFjixCCx1UYJf7n8fkMhLMVxcMrFIOQFg+RjBS4m4f7xRYpKWhCKjCa+VGq4YJBySBGSYhBlLYiErCxRYSQpl9MX9+KO6y6ItThpRrSCaAHccr2YElwNA8P5SJphy9ZBKHoOqeoOxO2llGGu6MTWcFbBGEPBcSNxHG6RcQulVLvvknUQUqsN/ryUBSGOQ273LZLUpC4pSC0TC1InKBIgvB+JNTr+GLkwlI/Fx5CT6iAsyc2VMo1o9bQiRsLdUmGQurx4Iz81VjyO+Di5DiIpzVW8b60vXlv/CqY5jIWzxkq7uXJEC6KrzSrpdmlmRAHLLYMkZce3l7Igxgp2ydc5tj9jLypcTNySMX3Xh7JQrpI6CEkxlBqXuDKZ3M0VAHrbUwCqaPedsF88SJ0sJkpVi4ukk4LUilYbeb9BYLRnE0X9+wphbkhuqUp9/FxJVRWkNtRprkSUqDxaEa0gmhxRuCS32lC/d/9wPphNdbWlWtaCEAU0F8zlLIhSGVtjhcosCH6sSJBaikGYRrJ1Vso/L7f7DiyIEuMuRsYRr6Q+tr/TG1OCYJJdTKVaclQqaEMXU+IuAAQXk3QsLlDFz4NbENExJNdBcGtCjltUWmfA9Z8ySJ1gBRgG4fRFPTh+XjyOEHZ6bX3x2prtPWcQogxIbvet/nUOjhcDQdSZsVq2F5MoCMeLpWMQYbA3WQHkik7EpZFEoCCcqGDmxwA84eQpiGQLonQ3V66Eyls+tuT7jymIuZ1Ys+NwsgUR64OULECTupzKhFlMpb9bvF5ENTbLMKLtvv0GgXIxXLmaBZ7FpNqnFGI3WA6/5lLK8e4/vVi5nb9nOiiI1r+CaY4oXJJmaUmWxXDOFiwIqy5BasYY/vPxV7B/KFfzsSqlWE0MwiltQRT9mEIlNSFiUZq8bTywIAgZy1Qq31JLpRYDF5OUxVRiXLIlI3/uy7kFkWgZRJ+XEqBVWxBlmigmBakBzxUj10GYhnrdiFLj87KY1FZGKcJ6kHAbV54TqXbm7jTtYtJMOlEXk1p4qPRDNmViOFcM8vY7M1ZdXEwDw3l86ecb8YsX99R8rEoRrzvMYkqIQRRLWxBcsCdVWr+4cxBf+Ol6jOTtsJjNjs7cxeOYgQWRnOZaqpsrz3SqJPuKH4/IdzFJQnlOl9deumILokRsodSazyKVupjSCVk//LW4i8moWEFwtxXPYuJUGoMIA9LxNNqJCPnp5GJq/SuY5lRiQajWFJjbncFwzkbRcWEQ0JGpT5B6JO8tbTk+he6qosLFVM6CSJqJc8WQsx1l6/CfPLcL33lyO/74+2uEltyqGIT33/BbQKiD1NE4Q2ScUh1EJdlX/NztKdOzIKTstTDYmpDFVGEMQn6tEgsiyf3JSSqU845vKCyIaJC6J5uS3qOqg1AHr8vB75d428LU2epFpFYQmilDlP2JrTYUMmVuVwZDvospbRl1S3Md8wVsJUHeeiEKwmrqIF47NIbL//ER7Dw8Fns/Y+pjcEH3m22HS8YgOJ6LKSEGUUG776Kk0EpmMXEFkbFihXLHzOko26LaMEJBWmo/QO2PV8H7JpVbGS9VwmWTsihiBfM6CDGgPas9qiBUld40wSA1l+OWIkg9kWpnS2cxaaYKUQg4CSaESnHM7WrDcM4LUqdMI1GIVQtXEJWkidYL0UKqtA4iV3SwducRbB0YxfOvHQleHxNcS6prGPUtpILj4uBIIXYueQ1pL0htKo/Fx62aXYcupmosCG/f9rQZKZTb8LdX4d4bLinbYI4L/VKz+WDfCi0IuwILQlzsR6kgpJm2yoLoLmVBCIpPVUBXjsDyEs63aFYWK+Z24jhFllI50tPIgtBZTE1OxMWk+A0ypl44pr8rg7ztYqxgI2MZyKTUgdRqGSv4LqY6dUutBFtZB5GUxRS+vm/I68ckrqonWj4ql5uoQHYPeu9TuZg4pkFoz5iBMomMu0SQOuZicspbEFwYZ1MmBpw8bJfBMihoOZ7UYVQcK+AtVDNedEoKfl5A5rJKYxDJCkK0BuQ4CBDtXwR498IyKHJe1bKoHO4q9GIQohVQmYDmu4m797an8cD/+52K3i+jXUyaKUMMcKqCnUm/y7nd3voFh0YLgQWR5Hevhka4mGyHBcKsXJqraEHs8zOtVC4m8Vgio74CBIA9R7z3uywUgLKLyTIIvdkUjox7CsIrbPT+Sgap5VYbZa4LCJVIh+BiUrlaEusgfKEaBoxL//yTlgmNjql0JTXgCepSrSvkdFuVBRE/Zvgav71EpIxNlIMrkkoVSjm4a6lUGnGroBVEkxNxMSl+hEkzt35/gZuDowWkLQNtKdOvCq6Pgjg8WsBXf7lp0lxNewdzWL3tkNfuwnaDGWRQKJfgbgvbT4eKQbQgxgQFoLKCRvN2EBDdMxi+T5XyCnhCt7c9jcGxIlyX4cKbf4W71uwsW+DIx8l85RO22igVg4i6mByHRYQgF0jlLIikthf/fO0ZuO51xwTPj57dXvJ4QJium9TuhZ+3lPtLnmm7LJ6RlHQtQGhBGCS14KhQQPND1Uk/BNXVsmXUimgXU5MjyhZZ0BQdF995cpvyfdxne3CkEASpAc8FI3bFrBYuYB99aQAPbdqPi5bPwYXHzpnw8ZI4/ysPAQBOW9SDF3YOYk5nGiP50C2UJEjFeMGOg3EFEXExKYL2o3kHS/ra8eKuQeweDGs9Co6npFQupu5sCkM5G0fGi9gzmMOmvcMRt1KpIDXgfY6V1EFwF1N/l6f89w/npa6rpS0DvmvS8p+rzliIVWcsDJ4vnd2BTXuHK2q1oZq8mAbBcRlSJpW0IFSumHIKQmxP7gguplJdaJMIlFeFFkc5plMMomFXQERXE9FmItpCRJ9q1DiaHbeEoPnuk9vxlXs3Kd/X1ebp/kOjBaRNUUHUFocYzXtClQvA0UmIRRwaDf35L+wcBBAKP66gkiwI8fpeDRTEWDDLFK0GVTX1WMHG4r4sAGDPkVCxhFlHchYT0Osr49cOeec7PFaIKogSQWrAu5eVZDHt9sdz2sIeAN7KeSoXU5Jg5HEF3nq7nADlFkSpAHQpN5qoFMIMKnWhnOq9/Ho6M6XnsfzUfEU5TrVprvVaAjRlUqyqu1VpiIIgIhPAvwJ4I4CTAPweEZ3UiLE0O6VcFaJvXaa7zRNaI3kbKcsI3AqVKAjbcbF+96DytXHBRQNEXTYA8J5/fwo3/XxD2XOU4t518SK8vX48IYhBJATcxesbzttoSxnIFV0c9JVOWQui4KAnm0Zvewp7BAsicDHZcRcTd0m96iuIQ6OF2Cp4MgXbDQSoLSxKVOrz+dWm/Th6djsuWO5ZbNsOjErFZKWzmPhrgYuprILoAADsEhSl6joAtRIM1m0WYhCqc6pm2gaFLce728opCF5AOLEYhGl41hXVyYJISetntzKNuopzAWxhjL3CGCsAuAPAqgaNpakRZ2/yTG44Z8u7B3QJP6qMaQSzxkpiBj9/cQ/e/M0nIj54jmwxHBot4OV9wwCAdbsG8cy2Q/iPxz23F2MM96zdXXWcYsPuocTXuIspMQYhCdjzls0GAGzc4x1TzFIaLyiymPI2OtImZnekg6JAIKymLrpxFxPvosoVxBHZgpAUBI+r8LUkyqXeemN18OutB3HZ8XOxaJZn4Qzl7KpaVAN+zCSbQjZlor+rLXE/ILQgdhwcjWx/7C8vww//6DykTApXxlMoQXHNhGpiEEC091JXWyr2emScfd44Lzu+P5KpVemSnnLfp1pJmca0iD8AjVMQCwGIS3rt9LdFIKLriGgNEa0ZGBiYssE1E6JSkH+EQwoF0Z42YRkU+VGlrHDWOJpPViqcHQfHwJgXKJYZkxTEF366AW/4+mM4MJLH95/aEYwBAJ7Zdgh//qPn8OVfbCx7Tvn85Ugq+pMVxBtPOQqmQVi97RCAqACWs5hcl2G04KA9Y2G2H+QPjuuoXUxekDrlj9sTpIfGChgUVryTFcTBUU+BLOj1BL1ohSVZEA9v3o+87eLyE+eiI2NhdofXViPaEru8q4THTJ781OvxxlOOStwPAE5e0A0AuPKk6H5LZrfjwmPnIGUaynbfKSlYLrYPVwltlYvJFNaW7ixjQRzT34k1n70Cf3jh0uCcc6TPrxRmnd1BKZOmRQYT0DgFobp7sSkIY+xWxthKxtjK/v7+KRhW8yFOlGU/73AuvuzmvR+7BJu/9MaIBZEyDZw03/uxi0VjL+8bxpb9w7Fj8PRQvryniOxS4uw+Mo71ezy3VK7owHbcIJawr0xjv8dfHsAR4VzbD45G3AoXHTsb3//wuZH3JLfaiAr944/qwikLe/C9p3bgoY37Ss7WucLgFoQIb47nuCyI5wCeYOEuJq7YXjs0jsv/8dFgn7hr0LPMlvouHB7XUY2J86PVr2JBTxsu9N1Li/xZc7UWBG+J3deRLjvD7m1PY/OXrsYfXbJM+bplkHJNajlI254xS45NGaSm8PvdVUZBAJ5C8CqpvWPJCr4UlmHULUAN+Isb1SslqsE06ip2AlgsPF8EoLI1IGcY0V5M5V1MlumZyynTCFY7S5sGFvdlsbA3iye3HAj2fcPXH8MVX3ssdoz9w16Bmar4S7YgOLuPjOPVg2NBcdXB0UIwuyy1stb+oRzed9tqfM+3PvK2g91HxnHqop5gnytOnIeTF/RE3pe3Xaz2LRRRccqxiSV97Vh59CwMjhfx4dvXYO9gLsjikoUxr4HoyFiYpVAQRaEOgeMpCG9fHqSWkf3zPHa0dA5XEFEL4sWdg3jXv/0aX7nXs7z2D+fw+MsH8O5zFgeCdrHvZlIuk1nifhtSpk85MpaZ6JtPW4aQVixYEP795fe5N5sO2mCog9SqLCYj6E77zrMWVTxefm39nekye4YYRn0tiHefsxgff8OKuh2vkTRKQfwGwAoiWkZEaQDXArinQWNpakSlILuYjihm+OJMiKdDpiwDRIQLl8/G068cKln1CiBo5V2NBbFhzzCGcjZWLu0D4LmneJVwqsSP79kdhwEAWwdGAHizb5cBpwgKoTNjxX7ABdvFe259Cves3R1x58ixib6OND508TKcu8wb1+MvDwTWwZd+vhHf9dOE9w/n8MkfvwAA6MiYsd4/BTtc4pO70IBokHq3wiUHAE+/chB/d5+XbXb387sCZbjU9/HzuE7a8prW/d19m/Cb7Yfx74++gv3DucAyOWvJrOCYZx/tPX5VUEpBBW+J+92RNpFN12ct5JRpBMF4W2FBcGHN74+chhoeR53FtHROB7bc9Ea85fQFFY+Jf09md1RpQdRRQZyztA+/f97RdTteI2mIgmCM2QD+FMAvAWwEcCdjbH0jxtLsJAWpGWMYGMkHz1V55vP8auqM/4M9Y0kvBseL2DuUw5DCPcXhLSoOjcb3SbIgnt56EABwLlcQQ7mgF1EpC4IriG0HRrHtwGigKE5ZGCqIrrZUTLDkbTdYilX028uuJyLCwt4sbv/guUiZhKGcHXFZfP/pHWCM4XN3r8cjm704V3vawqz26Ay04LhBBlOnZEGkLSOiNGRyRRe3PLIVh0cL+Ngdz2P1tkPobrPQ65+DK93uthTytovtB0dx+uJeAMCPnnktSG9d0BsGld+1cjFk5ve04W/efBKuPDk5tnDr+1fio5cuT3y9GrwYRNzFJLea6PGVrUmUEINQ10EA8e/O9z98Lj57zYmJY+IWSm9H6cB25D1lai5mMg0rlGOM/QLALxp1/lYhKc318FgxEjBNWwbsghP5os/t9gRK8EP1Z3KjeVtpfQBenIMrnkOj+djrY3m1gli93QsCr1zqzWz3DeWEnvolFMSrnoJ4YecgLvuHR4JxnrmkN9inqy1uQahWmQM8BdGVsTAsBeOzaRN9HWnsG8rjzMWz8NI+TxFtHRjFhj1D2LQ3jMV0KBRE0WF4yc/WOv6ormB/brH1ZlOJypPzuODey9tuMHPm2VI9WQvbD45hOFfEO85ciDbLwL3r9gTFa/N7ssH7OzMWvv6e0yNuQCLChy5Wxws4ouKtFcskpYspIy2Yw+tEvNqG+HFUGT9JAvuSFf24ZEVyPHIk701q5M+vFPUOUk8npkckZRqTtOTowZGo8FYtyDLPT2Pkr3Hf+XDOjvjLbcfFB7+zGg9v3o9dR8YD4au0IIqls6DOXDwLpkG+i8kTHkltj0fzNtbtGoz0+l82pwO3f+hczOsOZ8udGaukkhkvOPj8Pevx6Z+8iLztBtcpryHwF1cej2tOm48vrDo52JZNmfi9W5/GtgNhKmdHxlMmItsGRvCeW58GAKw8OnT18FjkUT1twViT+PGzO4PHedsNZsdc6S7ozcJxGVwGLJndgctPnItNe4fx7I5D6MmmIrEPAHj7mYvwR5ccg0aRNo2w1YbCguB9mvjnsKivHYtntceOo3IhTaTNNuBNnIB4e/BSmHUOUk8ndKuNJieS5ipYDHI6ZFqR4shdTNzt0uULmNG8Hfi1uzIWVm8/hIc3D+DVQ2PYOhAKStmC+L/nduG1Q+NIm4ayDqG3PYWe9hQWzcpi3e4hXHCMV4OQlNHx1NaDKDoM7zlnMW597BV0Ziw8/IlLg9d5i/JO34LozFiR2gTOT1/Yje/+ejsAT0Avn9uJvUM53PjGEyL7vXvlYrzbd80s6WvHqYt6cN0lx+Bvf7YBB0fy2O7fE1kQA4jcFzHtkl/b5SfOw29fPYL2tKkcIwA89pLnwnrHWQvx+hPmBrECHhy/cPkcPP6yZ2UcPbsdZyzuwZd/sQkPbtyPE46qvu30ZCOmuYoKYsW8Tozk7cBa4GnAP/+zi6GSw6cv7sXX3n06Nu8bxr8/+goAdTpsJRzxFURvNRaEMfHzTXe0BdHkRFxMrISCsOLtE/gsnAebueAbydtBcDOTMnD/+n0AokJweX9HMBvj3PDfzwOIF+xdcMxsvP+Co3HzO04FALz9zIV47KUBbPCL01ST/+0HRvH3v9yEbMrEe87xhPb7L4gG9nisgP+XLQLOXWt2Yk5nGl2+AunJprD95mtw7blLlPsDwGOfvAz/+t6zcPriXvzPRy/EI395WRC8TptGzILgsRHAE4yn+q4arpCvOnkeAC8D7L4bLsEVJ85Tnnd+Txu+9u4z8ObTFgQWBM9iOluwTJb0tWN5fyeW+ZlOvGaimcimzSCTTvyeXnTsHDz5qdcHExqe5WUYlJgR9Y6zFuHPXx9m/kzUguCu096E74oKsdJbE0UriCq58uuP4g/+85kpO18kSC2mc0qFYioX01w/i4kriE5BQby8f8Q/jotfbdofzPKuOXU+Hvj463DRsXMiPZHEWbGYTQMA5x3Th79ddQquPmU+AOC95y2BQcBP13qZy6oOsh/5r2fx0r4RXHTsbCzv78RDf/E7+MSVx0f24ePtynhj601wGxwczWNedxsuPWGudy8mWMX6r79/Fs4+ehYW9GZjM1BRQQznbNxx3fn43z+5MLjvx87twlUnz8M/vecMnHBUN46d2xl5f9o08M6zFuHjVxwXbOPFVDyLSQx0z+3y8vrf6rtfmlF+HTevE5v3DsN1WWStEu6u4cojSbHLiN/diTaUfPNp3ndQvv+lWLl0VtC+RBNFu5iq5KV9I0GAcyoQOzuIqYRytk7gYhJmaLxYiFsCnUIMgrezKNguBobzeOfZC7Hr8Dj++HeOwYp5XejrSGNwvIii461Ix1NfP/eWk3BMfydWf3t1cJ5uqRXC3K429HVkcMCPk8gdUPcMjmPT3mFcsmIObnq7Z3XwnHcR7lpq89uEcAXB+ytxGPOso3OWzsJP1+4Ozlst5x8zG//z0QsBxOMmPLOLX29Hxoopyn9/38rg8Q1XrEB31sJvtnnuu/OXz8Y/vvv0yP4p3z31Wz+Tq7c9hR/+0XnYtHc4mGm/9YwF+OeHXg4SDpqJkxf04L+efhUv7hqMdNfl7hqeKZek2GXEGgmVm68SPnDhUvz++UdX1QvpgxeVDuzPZLSCaDIee2kAK5fOClYJcxLSXGUFwTNHRF8qD5zyGgD+o1u/axAjeRvzujPYP5wHwcv6+NLbTg3eGyiX0QLmdrcFzfKOn9cV9HWa1Z7C4bFibDlIAOjrSCUqCO5n/8w1J0aC0TKdGQudGSsQlr2+q8Lz+7s4dm4ntviWUGfGwsqjvet8YeeRxGNWisoVctGxs/G5t5yMFRXMTttSJv7k0mPxkdeeBQBkU+p+QwCwae8wVp2xAItmtWPRrPZI+/Tl/Z348UcuwAl+JXwzwVtxrPrXJyPbuWzmVmelCkK0IDomWKtBRNNiLehmQbuYqqDW1djK8dqhMbz/26vxibvWBtuSKqnlGMSqMxbiM2+K5of3ZFN4/JOX4fNv8bJ20paBtGXg6Ve8moWzlswCY16mlLykI19wiFdV7/dn0PN62pBNeYqGC3dVt00xzVB2Mf16ywH0d2VwfJn1fjszqUjNAs+n53LkTUIvoY6MheP9QG417oVqWDyrHcfN66qq6ydvP51NxQXe3K4M0qaBFXM78ZkSuf0rl/aVbXndCOT1mrlg5pYA/+pW6mIS3Wh8gqRpLPpTqILJXmaTu5DWC91Mo+tBhPtyCyJteU3NTl7QjfP8rCGRxX3RtMLOjIXdgzmkTQMnL+jGvev2AkCkvxAQVmEPjOTxl3etxV1+iua87ja0p0x8+k0noOgwbNq7WWlBiApCznjadnAMx1cgaN94ylHBLBUIA4/vWrkYK4+ehUuO68c3frXFvy6v388vb3gdZlfRZqEUczrTSJlG0PZ7yex4imY5uMtPVsCAZ6Vt+uLVLZtB05Yy8dbTF+AeP9aUNg0UHSdWU6BSjioo4mKqT7W3pja0BVEFYkuHyYBnUvAeQbmig3W7PGVBBDhCQIJbEHz2XmmhD5+JLurLIivM0jLSj5gHuAeG84Fy4KmmhkG47nXLAyWiaqYm9jKS11DYdXgcCyvIynnn2Yvw8TeEQV3uqnBchjeeOh/twpg7/Gs5/qiuqjp5lmL1p6/A45+8LHjOXVjVwD9TlYIAWj+98hu/d2bQCDJIlPAF/ZfffirOP6ZvQussaAuiOdAKogomW0FwdxIPwN717E58/cGXAHiplYdGC0GHy4KfxVSuV74Mj0Ms7M1GMkVkC4IL2YHhMDgr93A6d2kfrjhxLo6ZE3fp9AmtDsQYRK7o4MBIHgtnVZ+2ya+Vt6YwDArGPdGgZikMgyKtHk5fXH0VslFGQUwH+KSDt5Tn1/ze85bgjusumNAxtQXRHGgFUQWDY5OnIE7/wv344s+8ldi4BXFYSDNNmwbW7hzE1f/0OIDQguA/zqQFdGR4sdyCnmzQowmIC7Bs2kRXxsLAcD5YglNm6ZwO/OcHzlE2f0uKQfC+QpVYEDKqBXb4uafCR88F4ESo1M3SinBhLlsQNR1TWxBNwYz+FPYP5fDKgVGcr/Ddq5hMC2JwvIgHN+4HEAp/MebBG9/x5R8LsoKocK1p/iOe39sWsSDaFHnn/V0ZDIzkg2Dj3//uaRVfT1IMgo9/IhYEFxoRBZEycQTFSbEgOL937uKKvyMyvN1INj1952Lt/r3n38V6dIudzM9TUzkz+lN48zefwP7hPLbffE1F+8urhNWrwVdS++3xyOI2UQVQcFwQhbO3StaaBhA0sVsgu5gUM9w5XRkMDOeRK7p473lLgjYVlSBWIosupl2Ha7AgMvFV8fjMfDJdEl95R+WKUYZbT9PaxeQr7tMW9eAvrjxuwspUpFR3XM3UMX2nNRWwf7i6gipRQSQteVmKpFl+UnaU6hy8CVnedpGxDPzu2d5iKiceVVme/LB/DQt7s5FioiQL4sBwHrmig7Yq3Sti7ntRsiAMCms0qoE3euN1HUAoeCtZdawRcOtpOisIPts3DcLlJ86ry8RJWxDNwYxWENUyJCqIYmUzds66XYM47rP34sEN+2KvJa0TPa5oH80DtQXbRdo0cPUp87H95msqTsHk1a3zeyQXk0KA9Xd6FsR40anaRSIK7KIdWkgHRvLo68hUVenKWdzXjsc/eRluENpVcHdGs/qs+aRgOiuITt96U60WN1G0BdEcaAWBZBePjGhB5Kq0INb61b0PbapcQYhupU+/6QQYFFobedtRuoXKwZfunN+TjfQsyigqfbvavHUVHJdVbUH0d4UWgmhBcMtnoizua4/MUEMXU3MqCH7t0zlI3V5lHKyiYzapwp9paAWBMABcjoiCqNKC4JkdtqJx3WjCIjyi6+nNpy3AH164LLAq8r4FUS3ffO+ZuPv6i5BNm1KQOi7AxMygagOPPdkU1v7NlXj7mQsjQepCjQpChs/Mm7HSGJgZCoIr53oWkuoFfJqDmn6pRPQuIlpPRC4RrZReu5GIthDRZiK6Sth+NhG96L/2DZpIFU2dkS2IvO0o22qMRBaXr+7HwL/w8gL2QLgegExO+MGZBiGbNjBe9MaWt13lrL8c3W2pYDnLdIk0VyA6K5+ItdLTnkLGMiIWRMF2J+ReSiJwMTWpgghdTNN3LsZdTLlJ7jSgmXpq/dauA/AOAI+JG4noJADXAjgZwNUAvkVEXMLcAuA6ACv8v6trHEPNiHn64wUH53zpQfxy/d7YfmI31aotCF9BuAp31phCQRQdN/KDM4jQnrbguAwFxw1iELVQqlAOkCyICc6AxXWLAe+6JtrKWQVvgtesFkRhBmQx8fhPro4uJk1zUNMvlTG2kTG2WfHSKgB3MMbyjLFtALYAOJeI5gPoZow9xbwp+vcAvK2WMdQD0YIYLdgYytnYdSRXcr98hbOldbsGcXAkL1gQ8X1GFC6m0bwdUUIGhUImV3Dr4qopF6QWZ+UTnQGnhGUpAS+rp74KwvTvTXPO0Ln1VM9rbjb49yRXZk1uTesxWdOuhQCeFp7v9LcV/cfy9oZiuy7ufn4XetvTONHvCKqa6TsuA5HXpbLS2dIffmc1fvfsxUHTOaUFoQhS3/7rHRHXk2lQMIsfLzpekLqGyl6gvAUh1hZM2IKwKBKDKNqsru2YVy7tw+7B3IT6/UwFQVPFOrrVmo3JiEFomoOyCoKIHgRwlOKlzzDG7k56m2IbK7E96dzXwXNHYcmS5OUja8V2GG55ZCsWzcrii287xduWoCA60t6ylpVaEKN5B+MFO7AgVAFx1RrGvAcTxzBIaDVho2C7NWd6cEGdNg1l07jOiAUxMQWRNqMxiLzjoiddXf+oUrzl9AXKRe+bBX7tqWlsQYgTF830ouy3ljF2BWPsFMVfknIAPMtALLtdBGC3v32RYnvSuW9ljK1kjK3s7+8vN1QljstiC9ao9nEZg+OywI0kr7vMt3EhnbNdvDIwUjZY7bgMDguPqxrKWAWmuUEUCGnPgqjdxZQxveMlBbs76qAgUqYBl4XuuXrETlqJP7/cW2d5dkd9WpA3I9y9V48g9QXHzA6KPzWNZ7J+qfcAuJaIMkS0DF4wejVjbA+AYSI6389eej+AUoqmZq78+qO44Y7nY9vFnO2i4/qCPFzkRFUb4bihgjg4ksfr//FR/NWPXyh5/qLrwnFRUvEkZTGJmBRaEOMFxxO0dYpBJLmqOusUgwCAz92zDkXH9YPUzekOmgx+79wl2H7zNdM7SC31YqqFH113Pv7hXaeX31EzJdSa5vp2ItoJ4AIAPyeiXwIAY2w9gDsBbABwH4DrGWN8evFRAP8JL3C9FcC9tYyhHCnJxcERq5Q9C8KLD3BBrnQxsbCAZ8RfkP0hv8GeCtdl3optLguOx4//7I5D+LdHtwJILpQTMYwwpbNeFgRXEEnCvz4WhKcM/uvpV/H4ywMzzoKYCczpzOCmt5+C2/7wnEYPRVNnas1i+gljbBFjLMMYm8cYu0p47SbG2HLG2PGMsXuF7Wt8F9Vyxtifsklex9MySSnso51SQ/cSr1NQBZNdwYIY899fyu8aKAXGgq6e3IL46do9+OZDL3vHkrKYbvtApKQEgOdi4r7esTpZEKZBMA1KFP7igjwTDVKLY+xIW3Wvg9A0B79/3tETasCoaW6m/S/VMtQWhFh7YDthDMIVhLqM47KgrQDPPFIpH3F/QG1BFBw3OIccpD5uXlewEhnHJAosiFzRQcFxa85iArwZfpIlYvgryAG1xSA4RFT3OgiNRjN5TPtfasokZXsLMTBsu67nXmKhBaGKQbiMoc0yQASMVhBY5hlLYpCaK4qi7Qbb5CC1ZVJsxh7NYnKQLzp1EbRp0ygp/Hmqay2Fcpy8XR/LR6PRTA3T/peaGIMoRmMQTmBBhNtk+BoQGctQVj+r9uf/+Ri4hVJ03EBZyBaESaRsbRGkExacuhWcpS2zZCyjI1hOcqJB6tASKtiuN27tYtJoWoJp/0u1TANFZXuLUEEUHQbH9SwEt4QF4TAGw/fZJzXYE+EKgAlpntxCKTphAFtu6+3FBeIfDXcxjeZtFB1Wl6Z3Gau0BdGZsZCx1HUSlSAqg0BBaAtCo2kJpv0vNWVQECAWGRcsAFUdhNLF5DKYRGizzKotCNuNBr95dbHt91YSsQy10E6bBgwKu8rWx4IwSqawdqStmlI0xa6cYwUHjE3vqmKNZjrRnB3O6ohVYQxCzmJSBqmZ52JKWVRRcRt3K3lZTNEYBFdajstiffQNI0w9/fYfrsT8Hi87hPyGfUe4gqiDoP3ElcdjXncm8fWOjFVTq2rRfcYfT+eqYo1mOjEDFISBoqK9RURBOF72kstY0ObbUSgV1/XSTS3DqGhxFDGLyfHHwBVFMVAYrtqCsPg6BymcOD9cTrQtZQYr29XDgrjmtPklX1/e3xGsQjcRDo0WgsdcQWgLQqNpDaa9gvBcTHFhn5PqIEIXk7dNZUHYrgvTACyDkBcUhOsypY8+UgchuZYKZSyI+b1ZYMdhyD3ovAC5N/apqCf4q6tPUFZ/V8o7z1qEu5/fjRd3DWLYLy7UMQiNpjWY9r/UpCwmcY0C7loSewapu7kCpmHANCjSg2ksoVhOjEGIvYi884cxCHl8lmHgprefgs+95SSsPHpW9DWTggwsuVZiMjAMglWDIprVkcadf3wBAGAkXz/XmEajmXym/S/V8hescV2GL/5sA147NAYAgcsH4HUQYbDa26augzANT0iLs37edkOGWy4uY4FC4oqlKFgQsoIwyFv57YMXLYu1sU6ZU2tB1ANuMfDML21BaDStwbT/paZMgu26eHHXIG57Yhtu+O/nAUS7qtpOWAchp6OKOH4Wk2kYERcTnxmr9uf/uULi7+PKo2C7EWvGNKjk2gaWQUEGllXHdRUmE97Sg7uYWkWxaTQznWn/S7UMA7YTztKZIkuJZzFF6iCUQWov1mAZUQtiOMmCcHlxXGiRcAXBYxDcouDpoOUWaxctCMtonY8vbRqhi0lbEBpNSzDtf6kpk/w2056A5v50McbA1w0WXUyJaa7kzYZFF5RqwR9+PMBzMYkWA2OhwuLCnqeSmmVWRksJMYhWapudSRnaxaTRtBjT/peaMo1IIJgHSEUFwK0BL5PJ25a05KjpWxAiiTEIRRYT4FkPRdt7zquoeTFaucCzZRpBBlbrWRDcxdQ6ik2jmcm0joSZIJZJcFwWClVfOEUsCDsMGJdaD8L1W23IbqCkorkgSC3EIPj5uMLi1kA27X0UZhnh6VlE3BpqHUGbtozAFVePFiEajWbymfa/VB4Q5e0p+KzbTlAQPEahyv3nQWo5yKpqy+GdI6ykFvtB5e2wOG68aheToXzc7KQtIQZhTt/V1TSa6UTrSJgJwl02Q7moe8NxWeALLziekHYZEtt9M79OQmVBqCq1xWM4bjTonbfdwLoILQivZrFckFp0K01FHUS9SJsGckXvPqVaKHai0cxkpr+CkC0IHqRmYTfUSlxM/KlJ8RjEpj3DuPSrD2PdrkHc/fyuYLvYoE+2WGIuJr/3UjkFIQamW8mCEN1KulBOo2kNal2T+qtEtImIXiCinxBRr/DajUS0hYg2E9FVwvaziehF/7VvUKmk/zrALQbevyhlCBaEL6h46qkjpLnKQWquOEwjLsS///QObD84hjd/8wl87I7ng3iHEwlSh1ZGrugECoO7mNonYEG0koIQM5daadwazUym1l/qAwBOYYydBuAlADcCABGdBOBaACcDuBrAt4iIO55vAXAdgBX+39U1jqEkcgyCwwPOYk2DKywYFLcgvOeGIotJ5shYMXIMV7BMgGharByDKJ/FRMrHzY6oIHSQWqNpDWr6pTLG7meMcWn3NIBF/uNVAO5gjOUZY9sAbAFwLhHNB9DNGHuKedHg7wF4Wy1jKEcQg/AVhNgkj9c0FAQLwkkIUnNhzyupS8E7mDpCkFpsGMgVCBC6mHiaa7mFeUT3TKrF0lyDx1pBaDQtQT1/qR8CcK//eCGA14TXdvrbFvqP5e1KiOg6IlpDRGsGBgYmNCjZguDKwPZrGlKmgXxQZR1tjyESupjKWxCHxzwFYQsFeLbrostfvpO/DsTTXKerBZGxwswl7WLSaFqDsr9UInqQiNYp/lYJ+3wGgA3gB3yT4lCsxHYljLFbGWMrGWMr+/v7yw1VCReifE0DcW1o3iNIbJshNtETcQUFUa5WgVsQ4pKjtsvQ054CABwRFUQsBlH6I5kOMQhtQWg0rUHZ9SAYY1eUep2IPgDgzQAuZyzwy+wEsFjYbRGA3f72RYrtkwYXqNyC4EVmDgutAbHxnpjRJMJdT5VYEFwB2II14rgMve0p7Dw8jsMlXEzlZH402Ns6FgQft1XB/dNoNM1BrVlMVwP4KwBvZYyNCS/dA+BaIsoQ0TJ4wejVjLE9AIaJ6Hw/e+n9AO6uZQzl4EJUdjG5LoNBvHV3WAkdKhC1BcFXlCvFoVHvXI6w5GjRYejJxi2IWC+mshaE6GJqnZk4VxDze9tKdqvVaDTNQ60ryv0LgAyAB/wf/dOMsY8wxtYT0Z0ANsBzPV3PGONS+KMAvgsgCy9mcW/sqHWEu2F4kVYkSG3Elw+1E1xMEQvCFGsRKNKuGxBiENKSo50ZCwZFl+HkKbHtaV5JXfp6RKXQSjNxHqRe2Jtt8Eg0Gk2l1KQgGGPHlnjtJgA3KbavAXBKLeetBjmQWxRm9YafxZSvIAbhRLKYwmNmLBNFJ9qsL8xiijbrS5kGOjJWxMUkWxDlrJN0RDm1jgXBU1sXzWpv8Eg0Gk2ltI6EmSCyEBUVgGVSbHW4gtBgT4TXR8h1EKqcftmC4DEIyyB0ZixlkLqNWxAVdHMFvFXnyu3bVPhD1RaERtM6THsFIbthxCA0b5tRcOIWhFwoF7qYIFkQyQpCXN/adhhMw0CnZEHkinIMorJmfa0UfwCAgyPePTmqp63BI9FoNJXSWlJmAsQtiLAQzigRgxgvOPiXX70ca5thCL2YiNQpm4dGJAvCb7WRMgkdkgUxVpBiEGUVhPd6qpWsBwADw3kAQH9npsEj0Wg0lTLjFIRcSZ3kYhrO2/iH+1/CPz34MoCwstqrnfAb6xHFZvKWQRgLlEq45CgPindmrEhQm6e5ckuk0l5MrWZB8M9hfq+2IDSaVqHWLKamR844El1MhkEwmbpQjvPcq4eD/YFoN1dVTUR3NoUir9YWrJWiE8YgRMYLDtKWEQj8ii2IFlMQX3nHqfid9f04aX53o4ei0WgqpLWkzAQQ+xX1d2bCSmp/femUYSizmDib9g4DEFxMQpqrZcQXD+rJpgLXkuhi8oLiXhaTSMFxkTaNQNGUS13l52ulIjkA6O/K4H3nH61rIDSaFmLaKwjRgpjb3RYJQlum32pDEaTmDI4XsfPwWOhiEiwI1eJB3dlUrJ8TY95xPQsivppa2jJgUHjMSq6nlfowaTSa1mRmKYiuDIqOt6yoV0lNijqJeGuoi//uYQyNe7UOYgzCsyCi7/csCK6ExNiG68Ug2uJevZQZtUpKEVgQLdTJVaPRtCbTXsqIgnRut5dBU3BcOIwpYwiyBXH20bMAAAdHvSwc8T2mYcQK27rbLLgsvgYEY1C6mABP6HNLpPya1NqC0Gg0U8P0VxCWGIPwMmiKDoPjwq+kVhfScd5xlteNfDgnWhBcQcQF9az2NID4GhCAZx10KRRE2jIige+S1xPEIKb9R6fRaBrM9M9iEgRud9a73KLt+u2+AZI6kBftqFDv8wU+bxcuuqUsIxTsS/racf1ly8NGfZIFAXjCf3l/Z2yMacGCKGcZtGqaq0ajaT2mvZQRZ9q8qK0oupjkGIQbtSB6fQWhsiAMIxTUc7syeM85SwKFYbssVo2dMgkXLJ8dPOfeJM+C4C00pmehnEajaT2mvYIQXTZcWeRt1y9cM8rGILr8oDJfstQ0IKSkGrGYAD+f47BIkNp7zQAR4a6PXIBrz1mMTn+RIDEGUWmQWscgNBrNZDPtXUwivOV00fEVBMVdNbKLiQeVuQUhrgfB24UDccFtu64yBgEA5yztwzlL+/DL9XsBeG02TCHwXQqrRQvlNBpN6zGjpEzoYmJBJbXcS0m0IAwCOvy6BR6DEJccFWsi5CBzUgwi+tw7d0faigS+S6GD1BqNZqqYUVKGC9WC7QaV1GlJ0IpFc5bffRWQLQih1UbgYgprIwB1DEJ2H/Hn7RkzkjpbyTW00mJBGo2mNZkRLqbvfvAcLJvTgW0HRgH4dRB+87xSFoRpELIpE0RiDIIiGUeW1PqCC3jbiVsQ8rn4cTozlVsQXDFoC0Kj0Uw2M0JBXHr8XADArsPjADwlwNt9y5XQYiW1ZRCICB1pK5LFJGYcpSSBnRJjEFKQOmNF22xw66MjY1VvQeggtUajmWRqmoYS0ReJ6AUiep6I7ieiBcJrNxLRFiLaTERXCdvPJqIX/de+QVPYvU1Mc7X9Fd5i60UIjft4rKE9bUbqIMSMo9C1FO3GqopByBYE7+/ELYg3nnIUVvqV20m0ajdXjUbTetQqZb7KGDuNMXYGgJ8B+BsAIKKTAFwL4GQAVwP4FhHx6fMtAK4DsML/u7rGMVSMGINw/F5MstCOxiBCFxBf2EdstSEuP5qSeik9/cpB/Gb74cix5XPlit65OtImiAi3/MHZeN1x/SWvQXZpaTQazWRRk4JgjA0JTzsA8CnzKgB3MMbyjLFtALYAOJeI5gPoZow9xRhjAL4H4G21jKEa+KptI3nbr6SOB6nFwLIpBJGDbTELQq6D8I7313evj51fPhdfra5d0X4jibRksWg0Gs1kUXMMgohuAvB+AIMALvM3LwTwtLDbTn9b0X8sb0869nXwrA0sWbKk1qFijr/c5YGRQlBJLbtqRLeQJaShckyh86oYj7CEDq9JZFLRc+V9C0JeRKgUut23RqOZKspOQ4noQSJap/hbBQCMsc8wxhYD+AGAP+VvUxyKldiuhDF2K2NsJWNsZX9/addLJfRkU7AMwoGRPFy/WZ9qTWmOmGUUbJMK5cKYQPlme0kptaoOr0noLCaNRjNVlJVMjLErKjzWDwH8HMDn4FkGi4XXFgHY7W9fpNg+JRgGYU5nBgeG874FUVrQhnUK4W0yIq02wm6wldQnZBKUkWoRoSSICAt7s1g0K1vxezQajWYi1JrFtEJ4+lYAm/zH9wC4logyRLQMXjB6NWNsD4BhIjrfz156P4C7axlDtczpSmNgJB/0YioV7A0tCHUMQrQgktaU/vDFy4LHSdZKNRYEAPzqE7+DPzjv6Kreo9FoNNVSq5/iZt/d9AKAKwF8DAAYY+sB3AlgA4D7AFzPGHP893wUwH/CC1xvBXBvjWOoijmdGewf8hf/IUqc1QNCkFqMQRhyDCLaXVWMDbzr7EW4eMWc4LlcB8ERYxyVkLHMskuTajQaTa3UFKRmjL2zxGs3AbhJsX0NgFNqOW8tzOnMYN2uQQAo72Iy4zEIcR1q0zDCOojAggiPl7KMyApx9bIgNBqNZiqYcZHO/q4MDowUAEDZrE+EC/vubCrcRhTJWJKD1GIMQlwICCilICqPQWg0Gs1UMeMUBE91BTxhr7Ig+KSfu416RAUhLhgkLFlqKVxMKZMiCwDJWUycJNeTRqPRNJIZqCDSwWNVHQSAwC1kKBSEIbX4DiwIK57FlJIsCF39rNFoWokZ5/zu64gqCFWQmvdI4sK+uy0apGYQW234aa6GIgZhGpHurFPYdkqj0WhqZsYpiGxKSFlNsCB45R4PPPe0ixZEtGparmy2pJhDKaXwH+9fiVcPjU3sQjQajWaSmXEKok1QEAbF230DgG9ACBZEqCCICIavQsQ0V1UdhGVQJItJ5g0nzZvgVWg0Gs3kM+NiENl01IKopNWGGIMAPCVh+cHqoLuqEbcg5BiERqPRtBIzTkGIFoRqyVEgjDks6WsHEHaBjeyTTaG7LSUUyMUtiJRlRLKYNBqNppWYcS4mMQZhJMQgLl4xB28+bQEuP9FbiU4VR7jrIxdgblcGG/cMAxBjEOHx0iZpC0Kj0bQsM1pBWAkuJoMIbzp1fsnjLO/vBACcsrAb156zGGct8VaCM03ZxVSPUWs0Gs3UM+MUhJjWKlsQBgEuQ1Vuofa0hZvfeVrwXI5BaBeTRqNpVWbc/FZscmdKWUxil9aJooPUGo1mujDjFISIaUTjC3y2X8usP9p7ibQFodFoWpYZrSBk4R1aEPF9P3rpcpy8oLvsMSmyZrW2IDQaTesy42IQIrLwLuVi+qurT8BfXX1Cxcd1XKZdTBqNpqWZ0RZEkoKotWcSj0OkLYL2MGk0mlZFKwjxuS/NS7XHqOa4KdOo+VgajUbTKOqiIIjoE0TEiGiOsO1GItpCRJuJ6Cph+9lE9KL/2jeoAS1OA1eSdGpxKdFasEQFoV1MGo2mRalZQRDRYgBvAPCqsO0kANcCOBnA1QC+RUS8Qu0WANcBWOH/XV3rGKqFt9eQ13U+enaHt71mC8Lvz2QawTksrSg0Gk2LUQ8L4usAPomwSzYArAJwB2MszxjbBmALgHOJaD6AbsbYU4wxBuB7AN5WhzFUBa+elmf3J80vn6VUCUEMQnAxaUtCo9G0GjUpCCJ6K4BdjLG10ksLAbwmPN/pb1voP5a3Jx3/OiJaQ0RrBgYGahlqBF49LVsKJ87vAgDsPjJe0/HFNSL4KWYLCxVpNBpNK1A2zZWIHgRwlOKlzwD4NIArVW9TbGMltithjN0K4FYAWLlyZeJ+1ZJWLO4DACf6FsRL+4ZrOr4Yg2hPW/jbVSfj0uPm1nRMjUajmWrKKgjG2BWq7UR0KoBlANb6ceZFAH5LROfCswwWC7svArDb375IsX1KSXIxHTfPsyCWzemo6fim4GICgPdfsLSm42k0Gk0jmHChHGPsRQDBtJiItgNYyRg7QET3APghEX0NwAJ4wejVjDGHiIaJ6HwAzwB4P4Bv1nIBE0F2MT194+WwXRdtKRP3fuwSLJyVren4wTrVlo47aDSa1mVSKqkZY+uJ6E4AGwDYAK5njDn+yx8F8F0AWQD3+n9TCrcgHNfzWh3V0xa8dmIdAtViHYRGo9G0KnVTEIyxpdLzmwDcpNhvDYBT6nXeicAVRMFxyuw5MayEGIdGo9G0EjNyistn9nnbnZTjm4a3lGkDagA1Go2mbsxIBfH75y0BEK4KV28sI7rOhEaj0bQiM7Kb66ozFmLVGYnlFzVjGgRLxx80Gk2Lo6XYJGAZhg5QazSalkdLsUnAi0FoF5NGo2lttIKYBFImIWXpW6vRaFobLcUmAdMg7WLSaDQtz4wMUk827zt/KQ6O5hs9DI1Go6kJrSAmgYtXzCm/k0aj0TQ52g+i0Wg0GiVaQWg0Go1GiVYQGo1Go1GiFYRGo9FolGgFodFoNBolWkFoNBqNRolWEBqNRqNRohWERqPRaJQQY6zRY6gIIhoAsGOCb58D4EAdhzMVtOKYAT3uqaQVxwy05rhbccyAN+4Oxlj/RN7cMgqiFohoDWNsZaPHUQ2tOGZAj3sqacUxA6057lYcM1D7uLWLSaPRaDRKtILQaDQajZKZoiBubfQAJkArjhnQ455KWnHMQGuOuxXHDNQ47hkRg9BoNBpN9cwUC0Kj0Wg0VaIVhEaj0WiUTGsFQURXE9FmItpCRJ9q9HhKQUTbiehFInqeiNb42/qI6AEietn/P6sJxvltItpPROuEbYnjJKIb/fu/mYiuaqIxf56Idvn3+3kielOTjXkxET1MRBuJaD0Rfczf3uz3OmncTXu/iaiNiFYT0Vp/zF/wtzf7vU4ad/3uNWNsWv4BMAFsBXAMgDSAtQBOavS4Sox3O4A50ra/B/Ap//GnAPxdE4zzdQDOArCu3DgBnOTf9wyAZf7nYTbJmD8P4BOKfZtlzPMBnOU/7gLwkj+2Zr/XSeNu2vsNgAB0+o9TAJ4BcH4L3OukcdftXk9nC+JcAFsYY68wxgoA7gCwqsFjqpZVAG73H98O4G2NG4oHY+wxAIekzUnjXAXgDsZYnjG2DcAWeJ/LlJIw5iSaZcx7GGO/9R8PA9gIYCGa/14njTuJho+beYz4T1P+H0Pz3+ukcSdR9bins4JYCOA14flOlP6iNhoG4H4iepaIrvO3zWOM7QG8Hx6AuQ0bXWmSxtnsn8GfEtELvguKuw+absxEtBTAmfBmiC1zr6VxA018v4nIJKLnAewH8ABjrCXudcK4gTrd6+msIEixrZlzei9ijJ0F4I0Ariei1zV6QHWgmT+DWwAsB3AGgD0A/tHf3lRjJqJOAP8D4AbG2FCpXRXbmmncTX2/GWMOY+wMAIsAnEtEp5TYvSnGDCSOu273ejoriJ0AFgvPFwHY3aCxlIUxttv/vx/AT+CZfvuIaD4A+P/3N26EJUkaZ9N+Boyxff6PywXwHwhN7aYZMxGl4AnZHzDG/tff3PT3WjXuVrjfAMAYOwLgEQBXowXuNUccdz3v9XRWEL8BsIKIlhFRGsC1AO5p8JiUEFEHEXXxxwCuBLAO3ng/4O/2AQB3N2aEZUka5z0AriWiDBEtA7ACwOoGjC8G/+H7vB3e/QaaZMxERABuA7CRMfY14aWmvtdJ427m+01E/UTU6z/OArgCwCY0/71Wjruu93qqI+9T+QfgTfCyKLYC+Eyjx1NinMfAyy5YC2A9HyuA2QAeAvCy/7+vCcb6I3hmaxHejOTDpcYJ4DP+/d8M4I1NNObvA3gRwAv+D2d+k435Ynjm/wsAnvf/3tQC9zpp3E17vwGcBuA5f2zrAPyNv73Z73XSuOt2r3WrDY1Go9Eomc4uJo1Go9HUgFYQGo1Go1GiFYRGo9FolGgFodFoNBolWkFoNBqNRolWEBqNRqNRohWERqPRaJT8/6qDUzP0cKl+AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > 250:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
